diff --git a/modules/weko-workspace/tests/test_api.py b/modules/weko-workspace/tests/test_api.py index b399b452ca..7fcb62c165 100644 --- a/modules/weko-workspace/tests/test_api.py +++ b/modules/weko-workspace/tests/test_api.py @@ -5,7 +5,7 @@ from requests.exceptions import RequestException from requests.models import Response -from weko_workspace.api import JamasURL, CiNiiURL, JALCURL, DATACITEURL, arXivURL +from weko_workspace.api import JamasURL, CiNiiURL, JALCURL, DATACITEURL, arXivURL, extract_arxiv_id # class JamasURL: @@ -471,6 +471,42 @@ def test_get_data(self, app): result = datacite.get_data() assert result == {"response":"","error":"request error"} +# def extract_arxiv_id(value): +# .tox/c1/bin/pytest --cov=weko_workspace tests/test_api.py::test_extract_arxiv_id -vv -s --cov-branch --cov-report=term --basetemp=/code/modules/weko-workspace/.tox/c1/tmp +@pytest.mark.parametrize("value, expected", [ + # new style + ("2301.00001", "2301.00001"), + ("0704.0001", "0704.0001"), + ("2301.00001v2", "2301.00001v2"), + (" 2301.00001 ", "2301.00001"), + ("arXiv:2301.00001", "2301.00001"), + ("ARXIV:2301.00001v1", "2301.00001v1"), + # old style + ("hep-th/9901001", "hep-th/9901001"), + ("math.AG/0601001", "math.AG/0601001"), + ("arXiv:hep-th/9901001v3", "hep-th/9901001v3"), + # URL + ("https://arxiv.org/abs/2301.00001", "2301.00001"), + ("http://arxiv.org/abs/2301.00001v2/", "2301.00001v2"), + ("https://arxiv.org/pdf/2301.00001.pdf", "2301.00001"), + ("https://export.arxiv.org/abs/hep-th/9901001", "hep-th/9901001"), + # arXiv DOI + ("10.48550/arXiv.2301.00001", "2301.00001"), + ("https://doi.org/10.48550/arXiv.2301.00001", "2301.00001"), + # not arXiv ID + ("10.5109/16119", None), + ("https://doi.org/10.5109/16119", None), + ("10.1234/5678901", None), + ("2301.001", None), + ("23010.00001", None), + ("https://example.org/abs/2301.00001", None), + ("", None), + (None, None), +]) +def test_extract_arxiv_id(value, expected): + assert extract_arxiv_id(value) == expected + + # class arXivURL: class TestarXivURL: # def __init__(self, naid, timeout=None, http_proxy=None, https_proxy=None): @@ -492,6 +528,11 @@ def test__init__(self): assert arXiv._doi == "10.5109/16119" assert arXiv._timeout == 10 assert arXiv._proxy == {"http":"test_http_proxy","https":"test_https_proxy"} + assert arXiv._arxiv_id is None + + # arXiv ID + arXiv = arXivURL("arXiv:2301.00001v2") + assert arXiv._arxiv_id == "2301.00001v2" # def _create_endpoint(self): @@ -501,6 +542,10 @@ def test_create_endpoint(self): result = arXiv._create_endpoint() assert result == "10.5109/16119" + arXiv = arXivURL("hep-th/9901001") + result = arXiv._create_endpoint() + assert result == "hep-th/9901001" + # def _create_url(self): # .tox/c1/bin/pytest --cov=weko_workspace tests/test_api.py::TestarXivURL::test_create_url -vv -s --cov-branch --cov-report=term --basetemp=/code/modules/weko-workspace/.tox/c1/tmp @@ -509,6 +554,14 @@ def test_create_url(self): result = arXiv._create_url() assert result == "https://export.arxiv.org/api/query?search_query=doi:10.5109/16119" + arXiv = arXivURL("2301.00001") + result = arXiv._create_url() + assert result == "https://export.arxiv.org/api/query?id_list=2301.00001" + + arXiv = arXivURL("https://doi.org/10.48550/arXiv.2301.00001") + result = arXiv._create_url() + assert result == "https://export.arxiv.org/api/query?id_list=2301.00001" + # def url(self): # .tox/c1/bin/pytest --cov=weko_workspace tests/test_api.py::TestarXivURL::test_url -vv -s --cov-branch --cov-report=term --basetemp=/code/modules/weko-workspace/.tox/c1/tmp @@ -517,6 +570,10 @@ def test_url(self): result = arXiv.url assert result == "https://export.arxiv.org/api/query?search_query=doi:10.5109/16119" + arXiv = arXivURL("hep-th/9901001v1") + result = arXiv.url + assert result == "https://export.arxiv.org/api/query?id_list=hep-th/9901001v1" + # def _do_http_request(self): # .tox/c1/bin/pytest --cov=weko_workspace tests/test_api.py::TestarXivURL::test_do_http_request -vv -s --cov-branch --cov-report=term --basetemp=/code/modules/weko-workspace/.tox/c1/tmp diff --git a/modules/weko-workspace/tests/test_utils.py b/modules/weko-workspace/tests/test_utils.py index bd4ea16cc9..5ad2109141 100644 --- a/modules/weko-workspace/tests/test_utils.py +++ b/modules/weko-workspace/tests/test_utils.py @@ -3630,7 +3630,7 @@ def test_get_datacite_autofill_item(app): # .tox/c1/bin/pytest tests/test_utils.py::test_get_arXiv_record_data -vv -s --cov-branch --cov=weko_workspace --cov-report=term --basetemp=/code/modules/weko-workspace/tests/.tox/c1/tmp def test_get_arXiv_record_data(app, mocker_itemtype): mock_get_data = { - 'response': {"title": [{"@value": "test title", "@language": "ja"}]}, + 'response': {"feed": {"entry": {"id": "http://arxiv.org/abs/2301.00001v1", "title": "test title"}}}, 'error': '' } with patch('weko_workspace.utils.arXivURL.get_data', return_value=mock_get_data): @@ -3677,6 +3677,34 @@ def test_get_arXiv_record_data(app, mocker_itemtype): current_cache.delete('arXiv_data10.1234/test-doi1') result = get_arXiv_record_data('10.1234/test-doi', 1) assert result == [] + + # Test with no entry (no hit), error entry (invalid ID) and empty entry list + for response in [ + {"feed": {"title": "arXiv Query"}}, + {"feed": None}, + {}, + {"feed": {"entry": {"id": "http://arxiv.org/api/errors#incorrect_id_format_for_abc", "title": "Error"}}}, + {"feed": {"entry": []}}, + ]: + with patch('weko_workspace.utils.arXivURL.get_data', return_value={'response': response, 'error': ''}): + with patch('weko_workspace.utils.get_arXiv_data_by_key') as mock_by_key: + current_cache.delete('arXiv_data2301.000011') + result = get_arXiv_record_data('2301.00001', 1) + assert result == [] + mock_by_key.assert_not_called() + + # Test with multiple entries: use the first entry + response = {"feed": {"entry": [{"id": "http://arxiv.org/abs/2301.00001v1"}, {"id": "http://arxiv.org/abs/2301.00002v1"}]}} + with patch('weko_workspace.utils.arXivURL.get_data', return_value={'response': response, 'error': ''}): + with patch('weko_workspace.utils.get_arXiv_data_by_key', return_value={}) as mock_by_key: + with patch('weko_workspace.utils.get_arXiv_autofill_item'): + with patch('weko_workspace.utils.get_autofill_key_tree'): + with patch('weko_workspace.utils.build_record_model', return_value=[1]): + current_cache.delete('arXiv_data10.1234/test-doi1') + result = get_arXiv_record_data('10.1234/test-doi', 1) + assert result == [1] + api = mock_by_key.call_args[0][0] + assert api['response']['feed']['entry'] == {"id": "http://arxiv.org/abs/2301.00001v1"} # .tox/c1/bin/pytest tests/test_utils.py::test_get_arXiv_data_by_key -vv -s --cov-branch --cov=weko_workspace --cov-report=term --basetemp=/code/modules/weko-workspace/tests/.tox/c1/tmp @pytest.mark.parametrize('keyword, expected_result', [ diff --git a/modules/weko-workspace/weko_workspace/api.py b/modules/weko-workspace/weko_workspace/api.py index a0683311a5..d95a4eb695 100644 --- a/modules/weko-workspace/weko_workspace/api.py +++ b/modules/weko-workspace/weko_workspace/api.py @@ -20,6 +20,7 @@ """WEKO3 module docstring.""" +import re import traceback import requests import urllib.parse @@ -425,6 +426,42 @@ def get_data(self): response['error'] = str(e) return response + +ARXIV_ID_PATTERN = re.compile( + r'^(?:arxiv:)?' + r'(\d{4}\.\d{4,5}' # new style: 2301.00001 + r'|[a-z][a-z-]*(?:\.[a-z]{2})?/\d{7})' # old style: hep-th/9901001 + r'(?:v\d+)?$', + re.IGNORECASE) +"""arXiv identifier pattern (with optional "arXiv:" prefix and version).""" + +ARXIV_PREFIX_PATTERN = re.compile( + r'^(?:https?://(?:www\.|export\.)?arxiv\.org/(?:abs|pdf)/' + r'|(?:https?://(?:dx\.)?doi\.org/)?10\.48550/arxiv\.)', + re.IGNORECASE) +"""arXiv abstract/PDF URL or arXiv DOI (10.48550/arXiv.) prefix.""" + + +def extract_arxiv_id(value): + """Extract arXiv identifier from input value. + + Accepts a bare arXiv ID (e.g. "2301.00001", "hep-th/9901001"), + "arXiv:" prefixed ID, arxiv.org abs/pdf URL and arXiv DOI + (10.48550/arXiv.xxxx). + + :param value: input value. + :return: arXiv identifier, or None if value is not an arXiv ID. + """ + if not isinstance(value, str): + return None + value = ARXIV_PREFIX_PATTERN.sub('', value.strip().rstrip('/')) + if value.lower().endswith('.pdf'): + value = value[:-4] + if not ARXIV_ID_PATTERN.match(value): + return None + return re.sub(r'^arxiv:', '', value, flags=re.IGNORECASE) + + class arXivURL: """The Class retrieves the metadata from arXiv.""" @@ -443,6 +480,7 @@ def __init__(self, doi, timeout=None, http_proxy=None, https_proxy=None): """ if not doi: raise ValueError('DOI is required.') + self._arxiv_id = extract_arxiv_id(doi) self._doi = "/".join(doi.strip().strip("/").split("/")[-2:]) if timeout: self._timeout = timeout @@ -456,6 +494,8 @@ def _create_endpoint(self): :return: endpoint string. """ + if self._arxiv_id: + return self._arxiv_id endpoint_url = self._doi return endpoint_url @@ -467,6 +507,8 @@ def _create_url(self): """ endpoint = self._create_endpoint() + if self._arxiv_id: + return config.WEKO_WORKSPACE_ARXIV_ID_API_URL + endpoint url = config.WEKO_WORKSPACE_ARXIV_API_URL + endpoint return url diff --git a/modules/weko-workspace/weko_workspace/config.py b/modules/weko-workspace/weko_workspace/config.py index 4afe81185e..e67d6526cf 100644 --- a/modules/weko-workspace/weko_workspace/config.py +++ b/modules/weko-workspace/weko_workspace/config.py @@ -318,6 +318,9 @@ WEKO_WORKSPACE_ARXIV_API_URL = 'https://export.arxiv.org/api/query?search_query=doi:' """arXiv API URL""" +WEKO_WORKSPACE_ARXIV_ID_API_URL = 'https://export.arxiv.org/api/query?id_list=' +"""arXiv API URL for arXiv ID""" + WEKO_WORKSPACE_CINII_REQUIRED_ITEM = [ 'title', 'creator', diff --git a/modules/weko-workspace/weko_workspace/static/js/weko_workspace/workspace_register.js b/modules/weko-workspace/weko_workspace/static/js/weko_workspace/workspace_register.js index 6a7edf59f9..e876d7f447 100644 --- a/modules/weko-workspace/weko_workspace/static/js/weko_workspace/workspace_register.js +++ b/modules/weko-workspace/weko_workspace/static/js/weko_workspace/workspace_register.js @@ -2427,7 +2427,7 @@ function toObject(arr) { } // $('#meta-search').modal('show'); if (!$('#doiInput').val()) { - alert('please input the DOI'); + alert('please input the DOI or arXiv ID'); $scope.isButtonDisabled = false; return; } diff --git a/modules/weko-workspace/weko_workspace/templates/weko_workspace/item_register.html b/modules/weko-workspace/weko_workspace/templates/weko_workspace/item_register.html index d1726fb3ef..61e4e0e45b 100644 --- a/modules/weko-workspace/weko_workspace/templates/weko_workspace/item_register.html +++ b/modules/weko-workspace/weko_workspace/templates/weko_workspace/item_register.html @@ -127,7 +127,7 @@

-
diff --git a/modules/weko-workspace/weko_workspace/translations/en/LC_MESSAGES/messages.mo b/modules/weko-workspace/weko_workspace/translations/en/LC_MESSAGES/messages.mo index 73eff5e8ce..552917bb99 100644 Binary files a/modules/weko-workspace/weko_workspace/translations/en/LC_MESSAGES/messages.mo and b/modules/weko-workspace/weko_workspace/translations/en/LC_MESSAGES/messages.mo differ diff --git a/modules/weko-workspace/weko_workspace/translations/en/LC_MESSAGES/messages.po b/modules/weko-workspace/weko_workspace/translations/en/LC_MESSAGES/messages.po index 4b8d244d84..6f7eb11387 100644 --- a/modules/weko-workspace/weko_workspace/translations/en/LC_MESSAGES/messages.po +++ b/modules/weko-workspace/weko_workspace/translations/en/LC_MESSAGES/messages.po @@ -279,7 +279,7 @@ msgid "DOI Input" msgstr "" #: weko_workspace/templates/weko_workspace/item_register.html:130 -msgid "Please enter the DOI." +msgid "Please enter the DOI or arXiv ID." msgstr "" #: weko_workspace/templates/weko_workspace/item_register.html:136 diff --git a/modules/weko-workspace/weko_workspace/translations/ja/LC_MESSAGES/messages.mo b/modules/weko-workspace/weko_workspace/translations/ja/LC_MESSAGES/messages.mo index 460124acaa..6123e13672 100644 Binary files a/modules/weko-workspace/weko_workspace/translations/ja/LC_MESSAGES/messages.mo and b/modules/weko-workspace/weko_workspace/translations/ja/LC_MESSAGES/messages.mo differ diff --git a/modules/weko-workspace/weko_workspace/translations/ja/LC_MESSAGES/messages.po b/modules/weko-workspace/weko_workspace/translations/ja/LC_MESSAGES/messages.po index 9222ea7835..fcda57aa64 100644 --- a/modules/weko-workspace/weko_workspace/translations/ja/LC_MESSAGES/messages.po +++ b/modules/weko-workspace/weko_workspace/translations/ja/LC_MESSAGES/messages.po @@ -279,8 +279,8 @@ msgid "DOI Input" msgstr "DOI入力" #: weko_workspace/templates/weko_workspace/item_register.html:130 -msgid "Please enter the DOI." -msgstr "DOIを入力してください。" +msgid "Please enter the DOI or arXiv ID." +msgstr "DOIまたはarXiv IDを入力してください。" #: weko_workspace/templates/weko_workspace/item_register.html:136 msgid "Get" diff --git a/modules/weko-workspace/weko_workspace/translations/messages.pot b/modules/weko-workspace/weko_workspace/translations/messages.pot index a83cff112e..7dfe8901f6 100644 --- a/modules/weko-workspace/weko_workspace/translations/messages.pot +++ b/modules/weko-workspace/weko_workspace/translations/messages.pot @@ -278,7 +278,7 @@ msgid "DOI Input" msgstr "" #: weko_workspace/templates/weko_workspace/item_register.html:130 -msgid "Please enter the DOI." +msgid "Please enter the DOI or arXiv ID." msgstr "" #: weko_workspace/templates/weko_workspace/item_register.html:136 diff --git a/modules/weko-workspace/weko_workspace/utils.py b/modules/weko-workspace/weko_workspace/utils.py index 059a7162d2..22e35cae95 100644 --- a/modules/weko-workspace/weko_workspace/utils.py +++ b/modules/weko-workspace/weko_workspace/utils.py @@ -2395,6 +2395,15 @@ def get_arXiv_record_data(doi, item_type_id, exclude_duplicate_lang=True): if api_response["error"] \ or not isinstance(api_response['response'], dict): return result + # No hit returns a feed without entry. Invalid ID returns an error entry. + feed = api_response['response'].get('feed') or {} + entry = feed.get('entry') if isinstance(feed, dict) else None + if isinstance(entry, list): + entry = entry[0] if entry else None + feed['entry'] = entry + if not isinstance(entry, dict) \ + or '/api/errors' in str(entry.get('id', '')): + return result api_data = get_arXiv_data_by_key(api_response, 'all') items = ItemTypes.get_by_id(item_type_id) if items is None: