diff --git a/.github/workflows/medcat-trainer_ci.yml b/.github/workflows/medcat-trainer_ci.yml index 4b542f470..03845f85d 100644 --- a/.github/workflows/medcat-trainer_ci.yml +++ b/.github/workflows/medcat-trainer_ci.yml @@ -285,13 +285,15 @@ jobs: tags: cogstacksystems/medcat-trainer:test labels: ${{ steps.meta.outputs.labels }} cache-from: type=registry,ref=cogstacksystems/medcat-trainer:buildcache - cache-to: type=registry,ref=cogstacksystems/medcat-trainer:buildcache,mode=max - name: Run Django Tests run: | # run tests using the built image docker run --rm cogstacksystems/medcat-trainer:test /home/.venv/bin/python manage.py test + - name: Prune stale Docker build cache + run: docker system prune --all --force --volumes + - name: Push Docker MedCATtrainer image id: docker_build_push uses: docker/build-push-action@v7 diff --git a/medcat-trainer/webapp/api/api/admin/models.py b/medcat-trainer/webapp/api/api/admin/models.py index 50cae265d..ecc31e4d5 100644 --- a/medcat-trainer/webapp/api/api/admin/models.py +++ b/medcat-trainer/webapp/api/api/admin/models.py @@ -96,6 +96,10 @@ def _set_proj_from_group(self, proj: ProjectAnnotateEntities, group: ProjectGrou proj.project_status = group.project_status proj.concept_db = group.concept_db proj.vocab = group.vocab + proj.model_pack = group.model_pack + proj.deid_model_annotation = group.deid_model_annotation + proj.use_model_service = group.use_model_service + proj.model_service_url = group.model_service_url proj.require_entity_validation = group.require_entity_validation proj.train_model_on_submit = group.train_model_on_submit proj.add_new_entities = group.add_new_entities diff --git a/medcat-trainer/webapp/api/api/models.py b/medcat-trainer/webapp/api/api/models.py index 0b5d0c095..7cf2505da 100644 --- a/medcat-trainer/webapp/api/api/models.py +++ b/medcat-trainer/webapp/api/api/models.py @@ -499,7 +499,16 @@ class Meta: model_service_url = models.CharField(max_length=500, blank=True, null=True, help_text='URL of the remote MedCAT service API (e.g., http://medcat-service:8000)') + def _normalize_model_config(self): + """ModelPack and CDB/Vocab are mutually exclusive configuration options.""" + if self.model_pack_id: + self.concept_db = None + self.vocab = None + elif self.concept_db_id and self.vocab_id: + self.model_pack = None + def save(self, *args, **kwargs): + self._normalize_model_config() # If using remote model service, skip local model validation if not self.use_model_service: if self.model_pack is None and (self.concept_db is None or self.vocab is None): diff --git a/medcat-trainer/webapp/api/api/tests/test_models.py b/medcat-trainer/webapp/api/api/tests/test_models.py index d48a5f786..a0a8ded59 100644 --- a/medcat-trainer/webapp/api/api/tests/test_models.py +++ b/medcat-trainer/webapp/api/api/tests/test_models.py @@ -12,7 +12,9 @@ MetaAnnotation, MetaTask, MetaTaskValue, + ModelPack, ProjectAnnotateEntities, + ProjectGroup, Relation, Vocabulary, cdb_name_validator, @@ -103,6 +105,77 @@ def test_use_model_service_with_url_skips_model_validation(self): proj.save() self.assertIsNotNone(proj.id) + def test_save_with_model_pack_clears_stale_cdb_vocab(self): + mp = ModelPack(name='normalize-mp') + mp.save(skip_load=True) + proj = self._new_project(concept_db=self.cdb, vocab=self.vocab, model_pack=mp) + proj.save() + proj.refresh_from_db() + self.assertEqual(proj.model_pack_id, mp.id) + self.assertIsNone(proj.concept_db_id) + self.assertIsNone(proj.vocab_id) + + def test_save_amended_model_pack_with_stale_cdb_vocab_succeeds(self): + mp_a = ModelPack(name='normalize-mp-a') + mp_a.save(skip_load=True) + mp_b = ModelPack(name='normalize-mp-b') + mp_b.save(skip_load=True) + proj = self._new_project(concept_db=self.cdb, vocab=self.vocab, model_pack=mp_a) + proj.save() + proj.model_pack = mp_b + proj.save() + proj.refresh_from_db() + self.assertEqual(proj.model_pack_id, mp_b.id) + self.assertIsNone(proj.concept_db_id) + self.assertIsNone(proj.vocab_id) + + def test_save_with_cdb_vocab_clears_stale_model_pack(self): + mp = ModelPack(name='normalize-mp-clear') + mp.save(skip_load=True) + proj = self._new_project(model_pack=mp) + proj.save() + proj.concept_db = self.cdb + proj.vocab = self.vocab + proj.save() + proj.refresh_from_db() + self.assertIsNone(proj.model_pack_id) + self.assertEqual(proj.concept_db_id, self.cdb.id) + self.assertEqual(proj.vocab_id, self.vocab.id) + + +@override_settings(MEDIA_ROOT='/tmp/mct-tests-models') +class ProjectGroupModelConfigValidationTests(TestCase): + @classmethod + def setUpTestData(cls): + cdb = ConceptDB(name='pg_val_cdb', cdb_file='pg_val_cdb.dat') + cdb.save(skip_load=True) + vocab = Vocabulary(name='pg_val_vocab', vocab_file='pg_val_vocab.dat') + vocab.save(skip_load=True) + cls.cdb = cdb + cls.vocab = vocab + cls.dataset = create_dataset(name='pg_val_ds', file_name='pg_val_ds.csv') + + def test_save_amended_model_pack_clears_stale_cdb_vocab(self): + mp_a = ModelPack(name='pg-mp-a') + mp_a.save(skip_load=True) + mp_b = ModelPack(name='pg-mp-b') + mp_b.save(skip_load=True) + group = ProjectGroup( + name='pg-switch-model-pack', + dataset=self.dataset, + concept_db=self.cdb, + vocab=self.vocab, + model_pack=mp_a, + cuis='', + ) + group.save() + group.model_pack = mp_b + group.save() + group.refresh_from_db() + self.assertEqual(group.model_pack_id, mp_b.id) + self.assertIsNone(group.concept_db_id) + self.assertIsNone(group.vocab_id) + @override_settings(MEDIA_ROOT='/tmp/mct-tests-models') class AnnotatedEntitySaveUpdatesProjectTests(TestCase): diff --git a/medcat-trainer/webapp/frontend/src/components/common/ClinicalText.vue b/medcat-trainer/webapp/frontend/src/components/common/ClinicalText.vue index d036468cb..93777564d 100644 --- a/medcat-trainer/webapp/frontend/src/components/common/ClinicalText.vue +++ b/medcat-trainer/webapp/frontend/src/components/common/ClinicalText.vue @@ -74,8 +74,10 @@ export default { formattedText () { if (this.loading || !this.text || !this.ents) { return '' } if (this.ents.length === 0) { - let text = this.text.replace('&', '&').replace('<', '>').replace('>', '>') - text = text === 'nan' ? '' : text + // Must escape for vue3-runtime-template: raw '<' / '>' / '&' are compiled as Vue + // template HTML and throw (e.g. compiler-24 missing end tag), which can leave the + // annotator unable to render subsequent documents. + const text = this.text === 'nan' ? '' : _.escape(this.text) return this.addAnnos ? `
${text}
` : `
${text}
` } diff --git a/medcat-trainer/webapp/frontend/src/tests/components/ClinicalText.spec.ts b/medcat-trainer/webapp/frontend/src/tests/components/ClinicalText.spec.ts index 2253b3647..73f9a5eb0 100644 --- a/medcat-trainer/webapp/frontend/src/tests/components/ClinicalText.spec.ts +++ b/medcat-trainer/webapp/frontend/src/tests/components/ClinicalText.spec.ts @@ -387,5 +387,54 @@ describe('ClinicalText.vue', () => { const formattedText = vm.formattedText expect(formattedText).toContain('@contextmenu.prevent.stop') }) + + it('escapes HTML-like clinical markup when there are no annotations', async () => { + const { compile } = await import('@vue/compiler-dom') + const text = '18 >[ No Service Recorded ] [ LLAMS MDT ] status a & b' + const wrapper = mount(ClinicalText, { + props: { + ...defaultProps, + text, + ents: [] + }, + global: { + stubs: ['v-overlay', 'v-progress-circular', 'v-runtime-template', 'vue-simple-context-menu'] + } + }) + + const formattedText = (wrapper.vm as any).formattedText + expect(formattedText).toContain('>') + expect(formattedText).toContain('<unknown>') + expect(formattedText).toContain('&') + expect(formattedText).not.toContain('') + expect(() => compile(formattedText)).not.toThrow() + }) + + it('escapes HTML-like markup in unannotated segments when annotations exist', async () => { + const { compile } = await import('@vue/compiler-dom') + const text = 'before Sample after >[ note ]' + const ents = [{ + id: 1, + start_ind: 13, + end_ind: 19, + assignedValues: { 'Concept Anno': 'Correct' }, + manually_created: false + }] + const wrapper = mount(ClinicalText, { + props: { + ...defaultProps, + text, + ents + }, + global: { + stubs: ['v-overlay', 'v-progress-circular', 'v-runtime-template', 'vue-simple-context-menu'] + } + }) + + const formattedText = (wrapper.vm as any).formattedText + expect(formattedText).toContain('<tag>') + expect(formattedText).toContain('>') + expect(() => compile(formattedText)).not.toThrow() + }) }) diff --git a/medcat-trainer/webapp/frontend/src/tests/views/TrainAnnotations.spec.ts b/medcat-trainer/webapp/frontend/src/tests/views/TrainAnnotations.spec.ts index 53b671350..061d5e59b 100644 --- a/medcat-trainer/webapp/frontend/src/tests/views/TrainAnnotations.spec.ts +++ b/medcat-trainer/webapp/frontend/src/tests/views/TrainAnnotations.spec.ts @@ -52,7 +52,7 @@ describe('TrainAnnotations.vue fetchData', () => { if (url.startsWith('/api/project-annotate-entities/')) { return Promise.reject({ response: { status: 500, data: { message: 'Database unavailable' } } }) } - return new Promise(() => {}) + return new Promise(() => { }) }) wrapper.vm.fetchData() @@ -68,7 +68,7 @@ describe('TrainAnnotations.vue fetchData', () => { if (url.startsWith('/api/project-annotate-entities/')) { return Promise.reject({ response: { status: 401, data: { detail: 'Invalid token.' } } }) } - return new Promise(() => {}) + return new Promise(() => { }) }) wrapper.vm.fetchData() @@ -85,7 +85,7 @@ describe('TrainAnnotations.vue fetchEntities', () => { return Promise.reject({ response: { data: { message: 'Invalid token.' } } }) } // Stall created() lifecycle requests so they don't interfere with the test. - return new Promise(() => {}) + return new Promise(() => { }) }) wrapper.vm.project = project @@ -134,7 +134,7 @@ describe('TrainAnnotations.vue fetchEntities', () => { if (url.startsWith('/api/annotated-entities/')) { return Promise.resolve({ data: { results: [], previous: null, next: null } }) } - return new Promise(() => {}) + return new Promise(() => { }) }) wrapper.vm.fetchData() @@ -157,7 +157,7 @@ describe('TrainAnnotations.vue fetchEntities', () => { } }) } - return new Promise(() => {}) + return new Promise(() => { }) }) wrapper.vm.project = project @@ -173,8 +173,36 @@ describe('TrainAnnotations.vue fetchEntities', () => { expect(wrapper.vm.currentEnt.id).toBe(10) }) + it('resets annotation pagination state when switching documents', async () => { + const { wrapper, mockGet } = mountView((url) => { + if (url.startsWith('/api/annotated-entities/')) { + return Promise.resolve({ + data: { results: [], previous: null, next: null } + }) + } + return new Promise(() => { }) + }) + + wrapper.vm.project = { ...project, prepared_documents: [123, 456] } + wrapper.vm.currentDoc = { id: 123, text: 'first doc' } + wrapper.vm.ents = [{ id: 1, start_ind: 0, end_ind: 1, assignedValues: {} }] + wrapper.vm.nextEntSetUrl = '/api/annotated-entities/?project=1&document=123&page=2' + wrapper.vm.loadingMsg = null + + wrapper.vm.loadDoc({ id: 456, text: 'second doc with markup' }) + await flushPromises() + + expect(wrapper.vm.currentDoc.id).toBe(456) + expect(wrapper.vm.loadingMsg).toBeNull() + // Must request the new document, not continue the previous page URL. + const entCalls = mockGet.mock.calls.filter(c => String(c[0]).startsWith('/api/annotated-entities/')) + expect(entCalls.some(c => String(c[0]).includes('document=456'))).toBe(true) + expect(entCalls.some(c => String(c[0]).includes('page=2'))).toBe(false) + + }) + it('does not load a document when the auth cookie is missing', async () => { - const { wrapper, mockGet } = mountView(() => new Promise(() => {}), {}) + const { wrapper, mockGet } = mountView(() => new Promise(() => { }), {}) mockGet.mockClear() wrapper.vm.project = project @@ -183,5 +211,6 @@ describe('TrainAnnotations.vue fetchEntities', () => { expect(mockGet).not.toHaveBeenCalled() expect(wrapper.vm.currentDoc).toBeNull() + }) }) diff --git a/medcat-trainer/webapp/frontend/src/views/TrainAnnotations.vue b/medcat-trainer/webapp/frontend/src/views/TrainAnnotations.vue index 24b6012b1..db112e268 100644 --- a/medcat-trainer/webapp/frontend/src/views/TrainAnnotations.vue +++ b/medcat-trainer/webapp/frontend/src/views/TrainAnnotations.vue @@ -529,7 +529,12 @@ export default { } }) } + // Clear prior doc annotation state so we never render the new text with old ents + // (or continue paginating the previous document's annotated-entities URL). this.currentEnt = null + this.ents = null + this.nextEntSetUrl = null + this.loadingMsg = 'Loading document...' this.prepareDoc() }, prepareDoc () { @@ -620,7 +625,8 @@ export default { this.nextEntSetUrl = null this.loadingMsg = null this.errors.modal = true - this.errors.message = 'Failed to load document annotations. Please try again by refreshing the page.' + this.errors.message = `Failed to load document annotations for project ID ${this.projectId || ''}${this.currentDoc && this.currentDoc.id ? ', document ID ' + this.currentDoc.id : ''}. Please try again by refreshing the page. If the problem persists, please contact your administrator and provide them with this page's URL: ${window.location.href}` + if (err.response) { this.errors.message = err.response.data?.message || this.errors.message this.errors.description = err.response.data?.description || ''