Add support for HTTP headers in harvesters.

NatLibFi · Sep 16, 2024 · 7153a4d · 7153a4d
1 parent 27718ba
commit 7153a4d
Show file tree

Hide file tree

Showing 6 changed files with 37 additions and 19 deletions.
diff --git a/conf/datasources.ini.sample b/conf/datasources.ini.sample
@@ -42,11 +42,12 @@
 ; RecordManager settings:
 ; type                  Harvester type (HTTPFiles, OAI-PMH (default), SFX, SierraApi)
 ; url                   Harvesting URL
-; username              Username for HTTP authentication (OAI-PMH, HTTPFiled)
-; password              Password for HTTP authentication (OAI-PMH, HTTPFiled)
-; authType              Authentication type for HTTP authentication (OAI-PMH, HTTPFiled).
+; username              Username for HTTP authentication (OAI-PMH, HTTPFiles)
+; password              Password for HTTP authentication (OAI-PMH, HTTPFiles)
+; authType              Authentication type for HTTP authentication (OAI-PMH, HTTPFiles).
 ;                       Allowed options are "basic" (default), "digest" and "ntlm".
 ;                       Digest and NTLM authentication require the Curl adapter to be used (see HTTP section in recordmanager.ini).
+; headers               Array of HTTP headers to send with each request
 ; institution           The institution code mapped to the source (required)
 ; recordXPath           xpath expression used when loading records from a file to identify a single record (optional, e.g. //record)
 ; format                Record format in RecordManager
@@ -260,3 +261,10 @@
 ;metadataPrefix = oai_doaj
 ;format = doaj
 ;institution = DOAJ
+
+; Sample configuration with an API key in X-API-Key header
+;[priv]
+;url = ...
+;headers['X-API-Key'] = 'xyzzy'
+;metadataPrefix = marc21
+;format = marc
diff --git a/src/RecordManager/Base/Harvest/AbstractBase.php b/src/RecordManager/Base/Harvest/AbstractBase.php
@@ -5,7 +5,7 @@
  *
  * PHP version 8
  *
- * Copyright (c) The National Library of Finland 2011-2023.
+ * Copyright (c) The National Library of Finland 2011-2024.
  *
  * This program is free software; you can redistribute it and/or modify
  * it under the terms of the GNU General Public License version 2,
@@ -105,6 +105,13 @@ abstract class AbstractBase
      */
     protected $httpAuth = null;
 
+    /**
+     * HTTP headers to send with each request
+     *
+     * @var array
+     */
+    protected $httpHeaders = [];
+
     /**
      * Source ID
      *
@@ -267,6 +274,8 @@ public function init(string $source, bool $verbose, bool $reharvest): void
             $this->httpAuth = [$username, $password, $type];
         }
 
+        $this->httpHeaders = (array)($settings['headers'] ?? []);
+
         if (!empty($settings['preTransformation'])) {
             foreach ((array)$settings['preTransformation'] as $transformation) {
                 $style = new \DOMDocument();

diff --git a/src/RecordManager/Base/Harvest/GeniePlus.php b/src/RecordManager/Base/Harvest/GeniePlus.php
@@ -387,9 +387,8 @@ protected function sendRequest($path, $params): MessageInterface
         }
 
         $client = $this->httpService->createClient($apiUrl, $this->httpOptions);
-        $headers = [
-            'Accept' => 'application/json',
-        ];
+        $headers = $this->httpHeaders;
+        $headers['Accept'] = 'application/json';
         $url = $this->httpService->appendQueryParams($apiUrl, $params);
 
         if (null === $this->accessToken) {
@@ -497,8 +496,9 @@ protected function renewAccessToken()
     {
         // Set up the request:
         $apiUrl = $this->baseURL . '/_oauth/token';
-        $client = $this->httpService->createClient($apiUrl);
-        $headers = ['Accept' => 'application/json'];
+        $client = $this->httpService->createClient($apiUrl, $this->httpOptions);
+        $headers = $this->httpHeaders;
+        $headers['Accept'] = 'application/json';
         $params = [
             'client_id' => $this->oauthId,
             'grant_type' => 'password',

diff --git a/src/RecordManager/Base/Harvest/HTTPFiles.php b/src/RecordManager/Base/Harvest/HTTPFiles.php
@@ -257,7 +257,7 @@ protected function retrieveFileList()
     protected function retrieveFile($filename)
     {
         $url = $this->baseURL . $filename;
-        $request = $this->httpService->createClient($url, ['auth' => $this->httpAuth]);
+        $request = $this->httpService->createClient($url, ['auth' => $this->httpAuth, 'headers' => $this->httpHeaders]);
         $this->infoMsg("Sending request: $url");
 
         // Perform request and throw an exception on error:

diff --git a/src/RecordManager/Base/Harvest/OaiPmh.php b/src/RecordManager/Base/Harvest/OaiPmh.php
@@ -8,7 +8,7 @@
  * PHP version 8
  *
  * Copyright (c) Demian Katz 2010.
- * Copyright (c) The National Library of Finland 2011-2021.
+ * Copyright (c) The National Library of Finland 2011-2024.
  *
  * This program is free software; you can redistribute it and/or modify
  * it under the terms of the GNU General Public License version 2,
@@ -395,7 +395,10 @@ protected function normalizeDate($date)
     protected function sendRequest($verb, $params = [])
     {
         // Set up the request:
-        $client = $this->httpService->createClient($this->baseURL, ['auth' => $this->httpAuth]);
+        $client = $this->httpService->createClient(
+            $this->baseURL,
+            ['auth' => $this->httpAuth, 'headers' => $this->httpHeaders]
+        );
         $params['verb'] = $verb;
         $url = $this->httpService->appendQueryParams($this->baseURL, $params);
 

diff --git a/src/RecordManager/Base/Harvest/SierraApi.php b/src/RecordManager/Base/Harvest/SierraApi.php
@@ -337,9 +337,8 @@ protected function sendRequest($path, $params): MessageInterface
         }
 
         $client = $this->httpService->createClient($apiUrl, $this->httpOptions);
-        $headers = [
-            'Accept' => 'application/json',
-        ];
+        $headers = $this->httpHeaders;
+        $headers['Accept'] = 'application/json';
         $url = $this->httpService->appendQueryParams($apiUrl, $params);
 
         if (null === $this->accessToken) {
@@ -463,10 +462,9 @@ protected function renewAccessToken()
         // Set up the request:
         $apiUrl = $this->baseURL . '/' . $this->apiVersion . '/token';
         $client = $this->httpService->createClient($apiUrl);
-        $headers = [
-            'Accept' => 'application/json',
-            'Authorization' => 'Basic ' . base64_encode("{$this->apiKey}:{$this->apiSecret}"),
-        ];
+        $headers = $this->httpHeaders;
+        $headers['Accept'] = 'application/json';
+        $headers['Authorization'] = 'Basic ' . base64_encode("{$this->apiKey}:{$this->apiSecret}");
 
         // Perform request and throw an exception on error:
         for ($try = 1; $try <= $this->maxTries; $try++) {