From 14f2150e79ff3a601473a61881874a613dd81248 Mon Sep 17 00:00:00 2001 From: Masaharu Hayashi Date: Sun, 25 Oct 2026 17:58:28 +0000 Subject: [PATCH] =?UTF-8?q?feat(weko-workspace):=20arXiv=20ID=20=E3=81=8B?= =?UTF-8?q?=E3=82=89=E3=81=AE=E8=87=AA=E5=8B=95=E8=A3=9C=E5=AE=8C=E3=81=AB?= =?UTF-8?q?=E5=AF=BE=E5=BF=9C?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ワークスペースのアイテム登録画面で、DOI に加えて arXiv ID でも arXiv からメタデータを自動補完できるようにする。 - arXiv ID(新形式 2301.00001 / 旧形式 hep-th/9901001、arXiv: 接頭辞・ バージョン付き)、arxiv.org の abs/pdf URL、arXiv DOI(10.48550/arXiv.) を判定し、arXiv API の id_list で問い合わせる - 通常の DOI は従来どおり search_query=doi: で問い合わせる - arXiv の応答に entry が無い(該当なし)、またはエラー entry の場合は 補完なしとして扱う(従来は KeyError でエラー表示になっていた) - 入力欄のプレースホルダーとアラート文言を更新 Co-Authored-By: Claude Opus 5.5 --- modules/weko-workspace/tests/test_api.py | 59 +++++++++++++++++- modules/weko-workspace/tests/test_utils.py | 30 ++++++++- modules/weko-workspace/weko_workspace/api.py | 42 +++++++++++++ .../weko-workspace/weko_workspace/config.py | 3 + .../js/weko_workspace/workspace_register.js | 2 +- .../weko_workspace/item_register.html | 2 +- .../translations/en/LC_MESSAGES/messages.mo | Bin 7704 -> 7646 bytes .../translations/en/LC_MESSAGES/messages.po | 2 +- .../translations/ja/LC_MESSAGES/messages.mo | Bin 7927 -> 7958 bytes .../translations/ja/LC_MESSAGES/messages.po | 4 +- .../weko_workspace/translations/messages.pot | 2 +- .../weko-workspace/weko_workspace/utils.py | 9 +++ 12 files changed, 147 insertions(+), 8 deletions(-) diff --git a/modules/weko-workspace/tests/test_api.py b/modules/weko-workspace/tests/test_api.py index b399b452ca..7fcb62c165 100644 --- a/modules/weko-workspace/tests/test_api.py +++ b/modules/weko-workspace/tests/test_api.py @@ -5,7 +5,7 @@ from requests.exceptions import RequestException from requests.models import Response -from weko_workspace.api import JamasURL, CiNiiURL, JALCURL, DATACITEURL, arXivURL +from weko_workspace.api import JamasURL, CiNiiURL, JALCURL, DATACITEURL, arXivURL, extract_arxiv_id # class JamasURL: @@ -471,6 +471,42 @@ def test_get_data(self, app): result = datacite.get_data() assert result == {"response":"","error":"request error"} +# def extract_arxiv_id(value): +# .tox/c1/bin/pytest --cov=weko_workspace tests/test_api.py::test_extract_arxiv_id -vv -s --cov-branch --cov-report=term --basetemp=/code/modules/weko-workspace/.tox/c1/tmp +@pytest.mark.parametrize("value, expected", [ + # new style + ("2301.00001", "2301.00001"), + ("0704.0001", "0704.0001"), + ("2301.00001v2", "2301.00001v2"), + (" 2301.00001 ", "2301.00001"), + ("arXiv:2301.00001", "2301.00001"), + ("ARXIV:2301.00001v1", "2301.00001v1"), + # old style + ("hep-th/9901001", "hep-th/9901001"), + ("math.AG/0601001", "math.AG/0601001"), + ("arXiv:hep-th/9901001v3", "hep-th/9901001v3"), + # URL + ("https://arxiv.org/abs/2301.00001", "2301.00001"), + ("http://arxiv.org/abs/2301.00001v2/", "2301.00001v2"), + ("https://arxiv.org/pdf/2301.00001.pdf", "2301.00001"), + ("https://export.arxiv.org/abs/hep-th/9901001", "hep-th/9901001"), + # arXiv DOI + ("10.48550/arXiv.2301.00001", "2301.00001"), + ("https://doi.org/10.48550/arXiv.2301.00001", "2301.00001"), + # not arXiv ID + ("10.5109/16119", None), + ("https://doi.org/10.5109/16119", None), + ("10.1234/5678901", None), + ("2301.001", None), + ("23010.00001", None), + ("https://example.org/abs/2301.00001", None), + ("", None), + (None, None), +]) +def test_extract_arxiv_id(value, expected): + assert extract_arxiv_id(value) == expected + + # class arXivURL: class TestarXivURL: # def __init__(self, naid, timeout=None, http_proxy=None, https_proxy=None): @@ -492,6 +528,11 @@ def test__init__(self): assert arXiv._doi == "10.5109/16119" assert arXiv._timeout == 10 assert arXiv._proxy == {"http":"test_http_proxy","https":"test_https_proxy"} + assert arXiv._arxiv_id is None + + # arXiv ID + arXiv = arXivURL("arXiv:2301.00001v2") + assert arXiv._arxiv_id == "2301.00001v2" # def _create_endpoint(self): @@ -501,6 +542,10 @@ def test_create_endpoint(self): result = arXiv._create_endpoint() assert result == "10.5109/16119" + arXiv = arXivURL("hep-th/9901001") + result = arXiv._create_endpoint() + assert result == "hep-th/9901001" + # def _create_url(self): # .tox/c1/bin/pytest --cov=weko_workspace tests/test_api.py::TestarXivURL::test_create_url -vv -s --cov-branch --cov-report=term --basetemp=/code/modules/weko-workspace/.tox/c1/tmp @@ -509,6 +554,14 @@ def test_create_url(self): result = arXiv._create_url() assert result == "https://export.arxiv.org/api/query?search_query=doi:10.5109/16119" + arXiv = arXivURL("2301.00001") + result = arXiv._create_url() + assert result == "https://export.arxiv.org/api/query?id_list=2301.00001" + + arXiv = arXivURL("https://doi.org/10.48550/arXiv.2301.00001") + result = arXiv._create_url() + assert result == "https://export.arxiv.org/api/query?id_list=2301.00001" + # def url(self): # .tox/c1/bin/pytest --cov=weko_workspace tests/test_api.py::TestarXivURL::test_url -vv -s --cov-branch --cov-report=term --basetemp=/code/modules/weko-workspace/.tox/c1/tmp @@ -517,6 +570,10 @@ def test_url(self): result = arXiv.url assert result == "https://export.arxiv.org/api/query?search_query=doi:10.5109/16119" + arXiv = arXivURL("hep-th/9901001v1") + result = arXiv.url + assert result == "https://export.arxiv.org/api/query?id_list=hep-th/9901001v1" + # def _do_http_request(self): # .tox/c1/bin/pytest --cov=weko_workspace tests/test_api.py::TestarXivURL::test_do_http_request -vv -s --cov-branch --cov-report=term --basetemp=/code/modules/weko-workspace/.tox/c1/tmp diff --git a/modules/weko-workspace/tests/test_utils.py b/modules/weko-workspace/tests/test_utils.py index bd4ea16cc9..5ad2109141 100644 --- a/modules/weko-workspace/tests/test_utils.py +++ b/modules/weko-workspace/tests/test_utils.py @@ -3630,7 +3630,7 @@ def test_get_datacite_autofill_item(app): # .tox/c1/bin/pytest tests/test_utils.py::test_get_arXiv_record_data -vv -s --cov-branch --cov=weko_workspace --cov-report=term --basetemp=/code/modules/weko-workspace/tests/.tox/c1/tmp def test_get_arXiv_record_data(app, mocker_itemtype): mock_get_data = { - 'response': {"title": [{"@value": "test title", "@language": "ja"}]}, + 'response': {"feed": {"entry": {"id": "http://arxiv.org/abs/2301.00001v1", "title": "test title"}}}, 'error': '' } with patch('weko_workspace.utils.arXivURL.get_data', return_value=mock_get_data): @@ -3677,6 +3677,34 @@ def test_get_arXiv_record_data(app, mocker_itemtype): current_cache.delete('arXiv_data10.1234/test-doi1') result = get_arXiv_record_data('10.1234/test-doi', 1) assert result == [] + + # Test with no entry (no hit), error entry (invalid ID) and empty entry list + for response in [ + {"feed": {"title": "arXiv Query"}}, + {"feed": None}, + {}, + {"feed": {"entry": {"id": "http://arxiv.org/api/errors#incorrect_id_format_for_abc", "title": "Error"}}}, + {"feed": {"entry": []}}, + ]: + with patch('weko_workspace.utils.arXivURL.get_data', return_value={'response': response, 'error': ''}): + with patch('weko_workspace.utils.get_arXiv_data_by_key') as mock_by_key: + current_cache.delete('arXiv_data2301.000011') + result = get_arXiv_record_data('2301.00001', 1) + assert result == [] + mock_by_key.assert_not_called() + + # Test with multiple entries: use the first entry + response = {"feed": {"entry": [{"id": "http://arxiv.org/abs/2301.00001v1"}, {"id": "http://arxiv.org/abs/2301.00002v1"}]}} + with patch('weko_workspace.utils.arXivURL.get_data', return_value={'response': response, 'error': ''}): + with patch('weko_workspace.utils.get_arXiv_data_by_key', return_value={}) as mock_by_key: + with patch('weko_workspace.utils.get_arXiv_autofill_item'): + with patch('weko_workspace.utils.get_autofill_key_tree'): + with patch('weko_workspace.utils.build_record_model', return_value=[1]): + current_cache.delete('arXiv_data10.1234/test-doi1') + result = get_arXiv_record_data('10.1234/test-doi', 1) + assert result == [1] + api = mock_by_key.call_args[0][0] + assert api['response']['feed']['entry'] == {"id": "http://arxiv.org/abs/2301.00001v1"} # .tox/c1/bin/pytest tests/test_utils.py::test_get_arXiv_data_by_key -vv -s --cov-branch --cov=weko_workspace --cov-report=term --basetemp=/code/modules/weko-workspace/tests/.tox/c1/tmp @pytest.mark.parametrize('keyword, expected_result', [ diff --git a/modules/weko-workspace/weko_workspace/api.py b/modules/weko-workspace/weko_workspace/api.py index a0683311a5..d95a4eb695 100644 --- a/modules/weko-workspace/weko_workspace/api.py +++ b/modules/weko-workspace/weko_workspace/api.py @@ -20,6 +20,7 @@ """WEKO3 module docstring.""" +import re import traceback import requests import urllib.parse @@ -425,6 +426,42 @@ def get_data(self): response['error'] = str(e) return response + +ARXIV_ID_PATTERN = re.compile( + r'^(?:arxiv:)?' + r'(\d{4}\.\d{4,5}' # new style: 2301.00001 + r'|[a-z][a-z-]*(?:\.[a-z]{2})?/\d{7})' # old style: hep-th/9901001 + r'(?:v\d+)?$', + re.IGNORECASE) +"""arXiv identifier pattern (with optional "arXiv:" prefix and version).""" + +ARXIV_PREFIX_PATTERN = re.compile( + r'^(?:https?://(?:www\.|export\.)?arxiv\.org/(?:abs|pdf)/' + r'|(?:https?://(?:dx\.)?doi\.org/)?10\.48550/arxiv\.)', + re.IGNORECASE) +"""arXiv abstract/PDF URL or arXiv DOI (10.48550/arXiv.) prefix.""" + + +def extract_arxiv_id(value): + """Extract arXiv identifier from input value. + + Accepts a bare arXiv ID (e.g. "2301.00001", "hep-th/9901001"), + "arXiv:" prefixed ID, arxiv.org abs/pdf URL and arXiv DOI + (10.48550/arXiv.xxxx). + + :param value: input value. + :return: arXiv identifier, or None if value is not an arXiv ID. + """ + if not isinstance(value, str): + return None + value = ARXIV_PREFIX_PATTERN.sub('', value.strip().rstrip('/')) + if value.lower().endswith('.pdf'): + value = value[:-4] + if not ARXIV_ID_PATTERN.match(value): + return None + return re.sub(r'^arxiv:', '', value, flags=re.IGNORECASE) + + class arXivURL: """The Class retrieves the metadata from arXiv.""" @@ -443,6 +480,7 @@ def __init__(self, doi, timeout=None, http_proxy=None, https_proxy=None): """ if not doi: raise ValueError('DOI is required.') + self._arxiv_id = extract_arxiv_id(doi) self._doi = "/".join(doi.strip().strip("/").split("/")[-2:]) if timeout: self._timeout = timeout @@ -456,6 +494,8 @@ def _create_endpoint(self): :return: endpoint string. """ + if self._arxiv_id: + return self._arxiv_id endpoint_url = self._doi return endpoint_url @@ -467,6 +507,8 @@ def _create_url(self): """ endpoint = self._create_endpoint() + if self._arxiv_id: + return config.WEKO_WORKSPACE_ARXIV_ID_API_URL + endpoint url = config.WEKO_WORKSPACE_ARXIV_API_URL + endpoint return url diff --git a/modules/weko-workspace/weko_workspace/config.py b/modules/weko-workspace/weko_workspace/config.py index 4afe81185e..e67d6526cf 100644 --- a/modules/weko-workspace/weko_workspace/config.py +++ b/modules/weko-workspace/weko_workspace/config.py @@ -318,6 +318,9 @@ WEKO_WORKSPACE_ARXIV_API_URL = 'https://export.arxiv.org/api/query?search_query=doi:' """arXiv API URL""" +WEKO_WORKSPACE_ARXIV_ID_API_URL = 'https://export.arxiv.org/api/query?id_list=' +"""arXiv API URL for arXiv ID""" + WEKO_WORKSPACE_CINII_REQUIRED_ITEM = [ 'title', 'creator', diff --git a/modules/weko-workspace/weko_workspace/static/js/weko_workspace/workspace_register.js b/modules/weko-workspace/weko_workspace/static/js/weko_workspace/workspace_register.js index 6a7edf59f9..e876d7f447 100644 --- a/modules/weko-workspace/weko_workspace/static/js/weko_workspace/workspace_register.js +++ b/modules/weko-workspace/weko_workspace/static/js/weko_workspace/workspace_register.js @@ -2427,7 +2427,7 @@ function toObject(arr) { } // $('#meta-search').modal('show'); if (!$('#doiInput').val()) { - alert('please input the DOI'); + alert('please input the DOI or arXiv ID'); $scope.isButtonDisabled = false; return; } diff --git a/modules/weko-workspace/weko_workspace/templates/weko_workspace/item_register.html b/modules/weko-workspace/weko_workspace/templates/weko_workspace/item_register.html index d1726fb3ef..61e4e0e45b 100644 --- a/modules/weko-workspace/weko_workspace/templates/weko_workspace/item_register.html +++ b/modules/weko-workspace/weko_workspace/templates/weko_workspace/item_register.html @@ -127,7 +127,7 @@

-
diff --git a/modules/weko-workspace/weko_workspace/translations/en/LC_MESSAGES/messages.mo b/modules/weko-workspace/weko_workspace/translations/en/LC_MESSAGES/messages.mo index 73eff5e8ce63e45de1bd8fda57d7a98f76ac4b97..552917bb9904346e2ab400cb65210ad75aa190a3 100644 GIT binary patch delta 1780 zcmYMzTS!z<6vpvojiZ@P-ZHNRDt3{)Kp|slO@>Niq9RzJMOa``n2(_}N)drUl!m>8 zkVO|Fg-MGj@x_#iE>t85MfTDKs)q!_qW)hVNXGf?vuB^P*IN6m^F?nr1>VNG&KZ9# z{HOANaIvob|J`$&#d3Kc`PehQmf&kl!uObk^SBh#!xqdzrmPS{(dSr>%edZw8_fdN z$e@sk9`xfA^kH(iSs6B<<_E9=zhNy_5=9;M<1YMxrC1eVR)J@+2&Yj2ClVdES~^By zHoDl~ybRdWN|3cyhJLKUXuN=m_$J2VD7x?kD#I!C;5&@Luc(Sy7hu)p~j=z*1}Qq?%u^~kfV0TZzq6?qpblhYW7=P?CuU@lIg0-Hkx@&|Q)EH`U? zHm<nRKm~Le)fG1}4IiO0_=MW{J1T)$%*7~P0ESkK>|s@? z^}A6S@5gH)W*w*sW(7!33%saOZ9yH$cFe+ksM2*i_Mz4fpaQ<`IE3181ZgXKfafsd?0f9LR;;`S|*0E4#NpoZPkcstJ_$C^QgAkM%oIv3stFZ)RFa~ zDsvIlRaa0)HHg~qE~-Lq-be!G{2368tw!Bggv!u|I@1cIT2_xL*#T5VI#6wO67|4S z$ct_lP#a!HJ#QFQsk_eg80!9UOjQ3rXQ0SGqcWL8s%9aagW762>TL5-f$c>F(u^#% zcGUV_Bssf^d~A#_b=4cx`dQSva3wQBd23Cs6c+A5{e-URWccs=vq`D1sG6i{LT+GsM760m97PKV=HRiQS{<*%*0XD z%k%-OaUHF)2HR0xbqRIf463U>q5=w|l^9xdnw9@+tMyDM13zlxT2u!0sM5EhD%6j9 z%ZE_w$50thqy-H+perSQ@n11_ Bq%Z&g delta 1819 zcmZA0Uq}>D6vy$SZvL%Zv$QlVQ4^8SQhYG&e~7t|T2_P_iY^M7VE7;uE=5A2J%|Ja zfzcmLFg92MD-$B92hEp^vSw6)cqjV;UezCIOYt)*;T*DKsukce zEI}8~w>=DartL*y?I3!w5##X|D&q-UiPPx9cc>N4p&P&Aa{Ps=Sp3qz70E;uH6QhV zg~(yMImPmPt7f1-tV5Nm(YbCx{>$2MHFlvg??n9=0$@!BeOSmr;QM497>P%pW_xK*h~rF8;(s%%%*PSc-~oKs~4fwFT!; z3mQNrIOwMSNeqUZzz3+6Jx2};apJ%23o4d?)rs_Pv$~#8!%k`zny_H-{n|?3iP4cs|(o;yM}77yQug# zj`OH3SVUDMhHRB!BI;#JMI}^-T9_AA(Hit?#Z3&9Q3t9-J*fM=s1ja6mGC-h;s`2k z42y9Bv+z6WrAnh!s<0Vz@D8fGLa2GEw2-YDWXQ6pK`ml1Dx8BSi z6_{a3n8gKi|zTi5>UMTba2Ie!qgPZU%*5PN=iRDptIhLT-twqJ7 z&N%L5+~x7Ghlk5}0H0$E7Ou)vd4){sn6n#cc7jgx&f;pkfZBM{ zji=oEmzc%=_SU_aL7M%fQ^tXc%tj$pd;_*(lk*(v?w_F&dVxy(qjMfrX;wHBFGVF( zgPI@z7w7(ac+f(pP#?`{RK{bdjVEy*KEds{h&!;F)im!oYQX_iQwH6xw0Zr2lT(a;_p@tDrL?EIME7(J{LX<&Wije53 zB`D}2Bzg*p>Qb-;fki}-kOWcq(4WV8i6WwZ;B{=nm#8oH538_*^0n@I z)OedSjVa=Ok4~D-3EYMEum=kgLB)Nj@$)!>Z*VVmaBll>1`pyQ_F;;w2XGp<;$zgt z|4|7?D4*P{5|xG)ohj!Is7imhIJ_oE zD2bY%#tQbgAv#)U40UNHP#K>^C6vSMcmp@!XY9l%uCNwvK#h00cnGz@0aRk6Eqa?Oec@^x)*08)$X*)G#Kuv7mke_3b_kpC3@V|U&L^mZ-XZtPKKPTxBPISqY)j~3 TIG!y^wYK{w<6hX0)kH!|)vav! diff --git a/modules/weko-workspace/weko_workspace/translations/ja/LC_MESSAGES/messages.po b/modules/weko-workspace/weko_workspace/translations/ja/LC_MESSAGES/messages.po index 9222ea7835..fcda57aa64 100644 --- a/modules/weko-workspace/weko_workspace/translations/ja/LC_MESSAGES/messages.po +++ b/modules/weko-workspace/weko_workspace/translations/ja/LC_MESSAGES/messages.po @@ -279,8 +279,8 @@ msgid "DOI Input" msgstr "DOI入力" #: weko_workspace/templates/weko_workspace/item_register.html:130 -msgid "Please enter the DOI." -msgstr "DOIを入力してください。" +msgid "Please enter the DOI or arXiv ID." +msgstr "DOIまたはarXiv IDを入力してください。" #: weko_workspace/templates/weko_workspace/item_register.html:136 msgid "Get" diff --git a/modules/weko-workspace/weko_workspace/translations/messages.pot b/modules/weko-workspace/weko_workspace/translations/messages.pot index a83cff112e..7dfe8901f6 100644 --- a/modules/weko-workspace/weko_workspace/translations/messages.pot +++ b/modules/weko-workspace/weko_workspace/translations/messages.pot @@ -278,7 +278,7 @@ msgid "DOI Input" msgstr "" #: weko_workspace/templates/weko_workspace/item_register.html:130 -msgid "Please enter the DOI." +msgid "Please enter the DOI or arXiv ID." msgstr "" #: weko_workspace/templates/weko_workspace/item_register.html:136 diff --git a/modules/weko-workspace/weko_workspace/utils.py b/modules/weko-workspace/weko_workspace/utils.py index 059a7162d2..22e35cae95 100644 --- a/modules/weko-workspace/weko_workspace/utils.py +++ b/modules/weko-workspace/weko_workspace/utils.py @@ -2395,6 +2395,15 @@ def get_arXiv_record_data(doi, item_type_id, exclude_duplicate_lang=True): if api_response["error"] \ or not isinstance(api_response['response'], dict): return result + # No hit returns a feed without entry. Invalid ID returns an error entry. + feed = api_response['response'].get('feed') or {} + entry = feed.get('entry') if isinstance(feed, dict) else None + if isinstance(entry, list): + entry = entry[0] if entry else None + feed['entry'] = entry + if not isinstance(entry, dict) \ + or '/api/errors' in str(entry.get('id', '')): + return result api_data = get_arXiv_data_by_key(api_response, 'all') items = ItemTypes.get_by_id(item_type_id) if items is None: