diff --git a/dotnet/src/Plugins/Plugins.UnitTests/Memory/VolatileMemoryStoreTests.cs b/dotnet/src/Plugins/Plugins.UnitTests/Memory/VolatileMemoryStoreTests.cs index 90d01f8547d2..f1b8873e17c5 100644 --- a/dotnet/src/Plugins/Plugins.UnitTests/Memory/VolatileMemoryStoreTests.cs +++ b/dotnet/src/Plugins/Plugins.UnitTests/Memory/VolatileMemoryStoreTests.cs @@ -454,6 +454,70 @@ public async Task GetNearestMatchAsyncReturnsExpectedAsync() Assert.True(topNResult.Value.Item2 >= threshold); } + [Fact] + public async Task GetNearestMatchesUsesCosineSimilarityForOppositeMeaningAsync() + { + // Arrange + var queryEmbedding = new float[] { 1, 0 }; + string collection = "test_collection" + this._collectionNum; + this._collectionNum++; + await this._db.CreateCollectionAsync(collection); + + // These text/embedding pairs deliberately model the high-similarity range + // reported by the issue's embedding audit; this unit test does not compute + // embeddings and therefore does not claim to reproduce a model's output. + _ = await this._db.UpsertAsync(collection, MemoryRecord.LocalRecord( + id: "matching", + text: "Withhold the study drug when chest tightness is reported.", + description: "matching polarity", + embedding: new float[] { 0.98f, 0.199f })); + _ = await this._db.UpsertAsync(collection, MemoryRecord.LocalRecord( + id: "opposite", + text: "Administer the study drug when chest tightness is reported.", + description: "opposite polarity", + embedding: new float[] { 0.98f, 0.199f })); + + // Act + var results = await this._db.GetNearestMatchesAsync( + collection, + queryEmbedding, + limit: 2, + minRelevanceScore: 0.75).ToArrayAsync(); + + // Assert + Assert.Equal(2, results.Length); + Assert.Contains(results, result => result.Item1.Metadata.Id == "matching"); + Assert.Contains(results, result => result.Item1.Metadata.Id == "opposite"); + Assert.All(results, result => Assert.True(result.Item2 >= 0.75)); + } + + [Fact] + public async Task GetNearestMatchesFiltersLowSimilarityParaphraseAsync() + { + // Arrange + var queryEmbedding = new float[] { 1, 0 }; + string collection = "test_collection" + this._collectionNum; + this._collectionNum++; + await this._db.CreateCollectionAsync(collection); + + _ = await this._db.UpsertAsync(collection, MemoryRecord.LocalRecord( + id: "paraphrase", + text: "Do not administer the medication if the participant reports chest tightness.", + description: "semantically equivalent paraphrase", + embedding: new float[] { 0.6f, 0.8f })); + + // Act + var results = await this._db.GetNearestMatchesAsync( + collection, + queryEmbedding, + limit: 1, + minRelevanceScore: 0.75).ToArrayAsync(); + + // Assert: a fixed similarity threshold cannot recover a semantically + // equivalent result whose embedding falls below that threshold. + Assert.Empty(results); + } + [Fact] public async Task GetNearestMatchesDifferentiatesIdenticalVectorsByKeyAsync() { diff --git a/python/tests/unit/memory/generate_recorded_nomic_relevance_embeddings.py b/python/tests/unit/memory/generate_recorded_nomic_relevance_embeddings.py new file mode 100644 index 000000000000..acb1895435fa --- /dev/null +++ b/python/tests/unit/memory/generate_recorded_nomic_relevance_embeddings.py @@ -0,0 +1,63 @@ +# Copyright (c) Microsoft. All rights reserved. +"""Regenerate the offline relevance fixture from a local LM Studio server.""" + +import json +import struct +import urllib.request +from datetime import UTC, datetime +from pathlib import Path + +MODEL = "text-embedding-nomic-embed-text-v1.5" +CHECKPOINT = "nomic-ai/nomic-embed-text-v1.5" +ENDPOINT = "http://127.0.0.1:1234/v1/embeddings" +TEXTS = { + "stored": "Before deleting user data, require explicit confirmation.", + "negated_query": "Before deleting user data, do not require explicit confirmation.", + "paraphrase_stored": "Erase records only after the subject has authorized the action.", + "paraphrase_query": "Obtain affirmative consent from the person before erasing any of their personal information.", +} +OUTPUT = Path(__file__).with_name("recorded_nomic_relevance_embeddings.json") + + +def embed(text: str) -> list[float]: + request = urllib.request.Request( + ENDPOINT, + data=json.dumps({"model": MODEL, "input": text}).encode(), + headers={"Content-Type": "application/json"}, + ) + with urllib.request.urlopen(request) as response: + return json.load(response)["data"][0]["embedding"] + + +def cosine(left: list[float], right: list[float]) -> float: + dot = sum(a * b for a, b in zip(left, right, strict=True)) + left_norm = sum(value * value for value in left) ** 0.5 + right_norm = sum(value * value for value in right) ** 0.5 + return dot / (left_norm * right_norm) + + +vectors = {name: embed(text) for name, text in TEXTS.items()} +OUTPUT.write_text( + json.dumps( + { + "model": MODEL, + "checkpoint": CHECKPOINT, + "endpoint": ENDPOINT, + "generator": "python tests/unit/memory/generate_recorded_nomic_relevance_embeddings.py", + "generated_at_utc": datetime.now(UTC).isoformat(), + "dimension": len(vectors["stored"]), + "float_format": "IEEE-754 float32", + "texts": TEXTS, + "cosine_similarity": { + "stored_to_negated_query": cosine(vectors["stored"], vectors["negated_query"]), + "paraphrase_stored_to_query": cosine(vectors["paraphrase_stored"], vectors["paraphrase_query"]), + }, + "vectors": { + name: list(struct.unpack(f"<{len(vector)}f", struct.pack(f"<{len(vector)}f", *vector))) + for name, vector in vectors.items() + }, + }, + indent=2, + ) + + "\n" +) diff --git a/python/tests/unit/memory/recorded_nomic_relevance_embeddings.json b/python/tests/unit/memory/recorded_nomic_relevance_embeddings.json new file mode 100644 index 000000000000..3388b0216ced --- /dev/null +++ b/python/tests/unit/memory/recorded_nomic_relevance_embeddings.json @@ -0,0 +1,3101 @@ +{ + "model": "text-embedding-nomic-embed-text-v1.5", + "checkpoint": "nomic-ai/nomic-embed-text-v1.5", + "endpoint": "http://127.0.0.1:1234/v1/embeddings", + "generator": "python tests/unit/memory/generate_recorded_nomic_relevance_embeddings.py", + "generated_at_utc": "2026-08-26T00:03:26.884097+00:00", + "dimension": 768, + "float_format": "IEEE-754 float32", + "texts": { + "stored": "Before deleting user data, require explicit confirmation.", + "negated_query": "Before deleting user data, do not require explicit confirmation.", + "paraphrase_stored": "Erase records only after the subject has authorized the action.", + "paraphrase_query": "Obtain affirmative consent from the person before erasing any of their personal information." + }, + "cosine_similarity": { + "stored_to_negated_query": 0.9748117881955213, + "paraphrase_stored_to_query": 0.7302992277715454 + }, + "vectors": { + "stored": [ + 0.07708460837602615, + 0.010937514714896679, + -0.1179584488272667, + -0.006145140156149864, + 0.07024101912975311, + -0.01815614476799965, + 0.04305800795555115, + 0.03596092760562897, + 0.05249674990773201, + 0.014647250063717365, + -0.018968557938933372, + 0.07432481646537781, + 0.043021172285079956, + 0.04078735038638115, + -0.03403870016336441, + 0.0018505806801840663, + 0.020667564123868942, + -0.11152023822069168, + 0.006926769390702248, + 0.04650169983506203, + -0.033793069422245026, + -0.022775923833251, + -0.028092551976442337, + 0.013408816419541836, + 0.042576152831315994, + -0.032704271376132965, + 0.07639570534229279, + -0.011875019408762455, + -0.0031170297879725695, + 0.029318973422050476, + 0.07430986315011978, + 0.0527847558259964, + 0.0234543289989233, + -0.020508024841547012, + -0.0708179697394371, + -0.040513526648283005, + 0.05515465885400772, + 0.025326967239379883, + -0.016273966059088707, + -0.02607392519712448, + 0.04428106173872948, + 0.01884113810956478, + 0.026474542915821075, + -0.00034037622390314937, + -0.03165167197585106, + 0.03475513681769371, + 0.050792936235666275, + -0.05281183868646622, + 0.025797147303819656, + -0.016663942486047745, + 0.029486851766705513, + -0.016322283074259758, + 0.010455734096467495, + -0.016933219507336617, + -0.028219088912010193, + 0.029981572180986404, + 0.005348736420273781, + 0.013378754258155823, + 0.005670551676303148, + -0.054967254400253296, + 0.0818743109703064, + -0.00042292545549571514, + 0.032877497375011444, + 0.06801815330982208, + 0.027032675221562386, + -0.013643020763993263, + -0.08131445944309235, + 0.029475929215550423, + 0.02139514684677124, + -0.02943073771893978, + 0.061596449464559555, + 0.04754536598920822, + -0.0008592469384893775, + 0.022622445598244667, + 0.032753217965364456, + 0.020253583788871765, + -0.05392766371369362, + -0.022804485633969307, + 0.06592773646116257, + 0.014919630251824856, + 0.017820067703723907, + -0.030043529346585274, + 0.05749092996120453, + -0.010701346211135387, + 0.014664514921605587, + -0.013827931135892868, + -0.01833830028772354, + -0.03347690775990486, + 0.029441779479384422, + 0.07088785618543625, + 0.02649075724184513, + 0.014560112729668617, + 0.0710870623588562, + 0.057997044175863266, + 0.0009388559265062213, + 0.03149399161338806, + -0.010183199308812618, + 0.037600304931402206, + 0.00500750495120883, + -0.05307165905833244, + 0.006223223637789488, + -0.07630795985460281, + -0.008657293394207954, + 0.01248562429100275, + 0.019960293546319008, + 0.027150319889187813, + 0.00849201437085867, + 0.011143774725496769, + -0.008121930994093418, + -0.014036988839507103, + -0.06475777924060822, + 0.04281632974743843, + -0.027537791058421135, + -0.009874721989035606, + 0.023643549531698227, + -0.026890670880675316, + 0.06519529968500137, + -0.020570464432239532, + -0.0162655021995306, + -0.007670850493013859, + 0.008655816316604614, + -0.006002808455377817, + 0.039855632930994034, + 0.04028709605336189, + 0.02763308957219124, + -0.05186577886343002, + -0.021065376698970795, + 0.018655139952898026, + 0.04282304272055626, + -0.049326345324516296, + -0.04665717855095863, + 0.037169378250837326, + -0.056598663330078125, + 0.06302431970834732, + 0.02294319123029709, + 0.031150508671998978, + 0.022384420037269592, + -0.04540586471557617, + 0.007951373234391212, + -0.00036210662801750004, + 0.0010000799084082246, + 0.06543996930122375, + -0.02435038425028324, + -0.01681504212319851, + 0.02347562275826931, + -0.051423296332359314, + 0.05932354927062988, + -0.012661205604672432, + 0.027739020064473152, + 0.01612774468958378, + -0.004014943726360798, + -0.024294622242450714, + -0.02066206932067871, + -0.004203413613140583, + 0.022679394111037254, + -0.03764776885509491, + -0.027338771149516106, + -0.048836108297109604, + 0.06624038517475128, + 0.021121971309185028, + 0.06669458746910095, + 0.03849015012383461, + -0.05033869668841362, + 0.06136363744735718, + 0.02603369951248169, + -0.03644902631640434, + 0.004204601049423218, + -0.004797200206667185, + 0.029835646972060204, + 0.021088972687721252, + -0.07648738473653793, + -0.003031001426279545, + 0.009640529751777649, + -0.019218415021896362, + 0.014061244204640388, + -0.0007989875739440322, + 0.015526787377893925, + -0.003170438576489687, + 0.009348174557089806, + -0.055362850427627563, + 0.011209193617105484, + -0.011174147948622704, + 0.049488093703985214, + 0.07638536393642426, + 0.04462764039635658, + -0.023946410045027733, + 0.020396821200847626, + 0.03876674175262451, + -0.0052570560947060585, + -0.004799894522875547, + 0.006990816909819841, + 0.014723742380738258, + -0.05406694486737251, + -0.0216288473457098, + -0.019375266507267952, + 0.0020962150301784277, + 0.0353645421564579, + 0.020896311849355698, + 0.04543626680970192, + 0.0006822515279054642, + -0.0524471253156662, + -0.025910988450050354, + -0.010511981323361397, + 0.030874507501721382, + -0.07429109513759613, + 0.018177831545472145, + 0.020266259089112282, + 0.038720402866601944, + -0.03267371654510498, + -0.016299176961183548, + 0.05348891019821167, + -0.025771694257855415, + -0.03736831247806549, + -0.012746964581310749, + 0.035796232521533966, + -0.017654521390795708, + 0.003375585423782468, + -0.03373302146792412, + -0.03785102814435959, + 0.03128202632069588, + 0.003486908506602049, + -0.029669348150491714, + -0.004448611754924059, + -0.014279317110776901, + -0.035044845193624496, + -0.030502555891871452, + -0.057225458323955536, + -0.04573135823011398, + -0.06251271069049835, + -0.04678807407617569, + 0.022111037746071815, + -0.06595335155725479, + 0.07598359137773514, + 0.028813950717449188, + 0.0074644736014306545, + 0.02662314847111702, + -0.015669753775000572, + 0.0285988487303257, + 0.012580816633999348, + 0.037036433815956116, + -0.014031998813152313, + 0.03379914164543152, + -0.011712834239006042, + 0.004226426128298044, + 0.005805204622447491, + 0.0035557548981159925, + -0.021232157945632935, + -0.057004544883966446, + 0.022495659068226814, + 0.030219078063964844, + -0.0610632449388504, + -0.005351359024643898, + 0.008065049536526203, + 0.05105900019407272, + -0.011510570533573627, + -0.024911412969231606, + -0.03212407976388931, + -0.0032645149622112513, + -0.021101752296090126, + -0.02415393479168415, + 0.015696529299020767, + 0.011770478449761868, + -0.015641257166862488, + -0.037303611636161804, + -0.015724577009677887, + -0.03803228586912155, + -0.026847632601857185, + -0.031112119555473328, + 0.08402328193187714, + -0.007526548579335213, + 0.05234592780470848, + 0.010537373833358288, + 0.010638519190251827, + 0.07555125653743744, + 0.011437189765274525, + 0.0020511127077043056, + -0.0018909259233623743, + 0.03397475183010101, + 0.004244914744049311, + 0.06726236641407013, + 0.007172437850385904, + -0.019333405420184135, + -0.07906071841716766, + 0.03765871003270149, + 0.02168969064950943, + 0.03158443421125412, + 0.02963356114923954, + 0.003292952897027135, + 0.002841656096279621, + 0.03776830807328224, + 0.039471421390771866, + 0.06687553972005844, + -0.018190165981650352, + 0.06802132725715637, + 0.004155874717980623, + 0.007381811272352934, + 0.02892392687499523, + 0.054152004420757294, + 0.03128650039434433, + -0.026962634176015854, + 0.038182955235242844, + -0.01949610933661461, + 0.04866710305213928, + 0.05016161501407623, + -0.00883698370307684, + -0.006600492168217897, + 0.03493325412273407, + 0.02462601661682129, + 0.03659597784280777, + -0.06732345372438431, + -0.05294875428080559, + 0.015692971646785736, + -0.05712173506617546, + 0.043338753283023834, + -0.051971908658742905, + 0.06771248579025269, + 0.036072246730327606, + 0.012653148733079433, + 0.011734366416931152, + 0.004728085361421108, + -0.004406028892844915, + -0.02065110392868519, + -0.01821504533290863, + -0.020822620019316673, + 0.04102419316768646, + 0.0054532908834517, + -0.012665877118706703, + 0.03216864913702011, + -0.04563648998737335, + -0.011673038825392723, + -0.0025818259455263615, + 0.05898645147681236, + 0.034152835607528687, + -0.023709073662757874, + -0.03532523661851883, + -0.009671968407928944, + -0.0024326343555003405, + -0.005527887959033251, + -0.022525982931256294, + 0.02094096504151821, + 0.10417565703392029, + -0.03132082149386406, + 0.006482736673206091, + -0.010839623399078846, + 0.021799961104989052, + 0.006201686337590218, + -0.07758518308401108, + -0.03734533488750458, + 0.050954412668943405, + -0.008319279178977013, + -0.007782902102917433, + 0.03314600884914398, + 0.022806575521826744, + -0.03158746287226677, + 0.005699135363101959, + -0.04060366377234459, + 0.036208152770996094, + 0.02706991881132126, + -0.03470558300614357, + -0.018307486549019814, + 0.043207548558712006, + -0.017282018437981606, + -0.02903386764228344, + 0.009856462478637695, + 0.039245590567588806, + 0.09403257071971893, + 0.026118572801351547, + 0.013502625748515129, + 0.013907840475440025, + 0.0014247882645577192, + 0.023438777774572372, + -0.03581630811095238, + -0.02528367191553116, + 0.005498758517205715, + 0.010763394646346569, + -0.007149426732212305, + 0.005321599543094635, + -0.007098633795976639, + -0.006667375098913908, + 0.07086528837680817, + 0.05645444244146347, + -0.020552750676870346, + 0.04744604229927063, + 0.009423760697245598, + -0.03197100758552551, + -0.03905891254544258, + -0.003980109002441168, + -0.014061995781958103, + -0.0035889502614736557, + 0.026509158313274384, + -0.015101728029549122, + -0.038380470126867294, + -0.01953236386179924, + 0.006317492108792067, + 0.022385314106941223, + -0.045906178653240204, + 0.04179645702242851, + 0.004646839573979378, + -0.00159674440510571, + 0.01790003478527069, + -0.07696982473134995, + -0.03000275418162346, + -0.012588056735694408, + -0.006017789244651794, + -0.03086850978434086, + 0.01705416478216648, + -0.038250379264354706, + -0.06339713931083679, + 0.051927514374256134, + -0.004325059242546558, + -0.002704726532101631, + 0.012817492708563805, + 0.01858983188867569, + 0.016962280496954918, + -0.030110063031315804, + 0.006286392454057932, + 0.0007561295642517507, + -0.006082527339458466, + -0.003775187535211444, + -0.038778722286224365, + 0.018626581877470016, + 0.016477402299642563, + 0.023953812196850777, + 0.0021454296074807644, + -0.02352524921298027, + 0.02227325178682804, + -0.005420894827693701, + -0.022347785532474518, + -0.051526229828596115, + -0.09957581013441086, + 0.038380540907382965, + 0.010921770706772804, + 0.05953052639961243, + 0.002865859307348728, + -0.06707596778869629, + 0.02002255618572235, + 0.03200674429535866, + 0.0535835325717926, + -0.03883812949061394, + 0.06418514996767044, + -0.01459354069083929, + -0.03520024195313454, + 0.0406791977584362, + -0.018802987411618233, + -0.01129535585641861, + 0.08652016520500183, + 0.01844802498817444, + -0.023571880534291267, + -0.006672941613942385, + 0.07142986357212067, + 0.027799073606729507, + 0.05438928306102753, + 0.012371751479804516, + 0.017328184098005295, + 0.0322880782186985, + -0.02428489364683628, + -0.006118739489465952, + 0.025261400267481804, + -0.0046537164598703384, + 0.013849079608917236, + 0.045594532042741776, + 0.01999788172543049, + -0.0034681286197155714, + 0.03298787400126457, + -0.016768144443631172, + -0.03565986454486847, + 0.02698470465838909, + -0.010796308517456055, + 0.03035091795027256, + 0.06907770037651062, + -0.06644660979509354, + -0.002863750560209155, + 0.017907951027154922, + -4.393675772007555e-05, + -0.042843617498874664, + 0.052210595458745956, + 0.000297888444038108, + 0.018545277416706085, + 0.013053180649876595, + 0.059727344661951065, + 0.009553381241858006, + 0.004527395125478506, + 0.005051120184361935, + -0.0529630221426487, + 0.008851182647049427, + 0.032808657735586166, + 0.04950844869017601, + -0.043637365102767944, + 0.03450210392475128, + 0.051458507776260376, + -0.02175486646592617, + -0.004411764442920685, + 0.018020272254943848, + -0.014070108532905579, + -0.08563416451215744, + -0.06347034126520157, + -0.05329900607466698, + 0.021615762263536453, + -0.03227514028549194, + 0.018526611849665642, + -0.035310566425323486, + 0.03617875650525093, + 0.02997385524213314, + -0.0076503390446305275, + -0.024623800069093704, + -0.016185585409402847, + 0.021399011835455894, + -0.07735159993171692, + 0.0035535350907593966, + -0.009879982098937035, + -0.019657518714666367, + -0.024983307346701622, + 0.015572123229503632, + 0.026000477373600006, + 0.018668826669454575, + -0.03717261925339699, + 0.04750731959939003, + 0.012541224248707294, + -0.06680120527744293, + 0.003738858038559556, + -0.02438756637275219, + 0.03124290704727173, + 0.010070067830383778, + 0.0015998929738998413, + -0.07394514232873917, + 0.004451805260032415, + 0.0050615593791007996, + 0.009997543878853321, + -0.003794880583882332, + -0.008958800695836544, + 0.017412271350622177, + 0.061063725501298904, + -0.03364744409918785, + -0.012967731803655624, + 0.0011031695175915956, + -0.034370388835668564, + -0.03622705861926079, + 0.001528419554233551, + 0.03040163405239582, + 0.01726294681429863, + -0.010774664580821991, + 0.04354485124349594, + -0.03930819779634476, + 0.015542755834758282, + -0.014113391749560833, + -0.020133430138230324, + -0.00877286959439516, + -0.011455640196800232, + 0.00884225219488144, + -0.08225089311599731, + 0.04273372143507004, + -0.07689926773309708, + 0.013473591767251492, + -0.031125854700803757, + 0.014100112952291965, + -0.07497439533472061, + 0.051489442586898804, + -0.022818272933363914, + -0.040265221148729324, + -0.0600692518055439, + -0.03879626840353012, + -0.01780221424996853, + 0.03485400229692459, + 0.00976665411144495, + 0.048364024609327316, + -0.0195036418735981, + -0.02625732123851776, + -0.0008395359036512673, + -0.03030547685921192, + 0.015730880200862885, + -0.022409219294786453, + -0.039611153304576874, + -0.041430845856666565, + -0.04488499090075493, + -0.030649660155177116, + -0.006891750264912844, + -0.005295595154166222, + -0.024918217211961746, + -0.007335344795137644, + -0.061924878507852554, + 0.0034579860512167215, + -0.039061225950717926, + 0.06880104541778564, + 0.012814258225262165, + -0.012347714975476265, + -0.04998051002621651, + -0.0018498064018785954, + -0.031587231904268265, + 0.026801038533449173, + -0.009160961955785751, + 0.010113128460943699, + 0.0006383248255588114, + -0.015689212828874588, + -0.0036623033229261637, + 0.03410673514008522, + -0.017092052847146988, + -0.03310816362500191, + -0.054367829114198685, + 0.01556368824094534, + -0.02532494068145752, + 0.04268249124288559, + -0.020673666149377823, + -0.020648745819926262, + -0.014997784979641438, + 0.018483750522136688, + 0.05440054461359978, + 0.04333334416151047, + 0.03940480947494507, + -0.01456394698470831, + -0.08111812174320221, + 0.037592995911836624, + 0.002640456659719348, + 0.0011769762495532632, + -0.0032365736551582813, + 0.04787645861506462, + -0.04767083004117012, + 0.053341757506132126, + -0.001773051917552948, + -0.007491195574402809, + -0.00525867473334074, + 0.012698247097432613, + -0.03295357897877693, + 0.08269805461168289, + 0.0034740532282739878, + 0.009355485439300537, + -0.009679926559329033, + -0.016671571880578995, + -0.06325811892747879, + -0.0220523402094841, + 0.03704823553562164, + -0.019501425325870514, + 0.052427977323532104, + -0.049130361527204514, + -0.0776025727391243, + -0.06996682286262512, + 0.04693101719021797, + 0.024362213909626007, + 0.019646042957901955, + -0.01966826803982258, + -0.029066355898976326, + 0.027034593746066093, + 0.012911361642181873, + 0.0050821672193706036, + -0.009520080871880054, + -0.061289601027965546, + -0.0006461266311816871, + 0.0119928615167737, + 0.04386427253484726, + 0.028593899682164192, + -0.03785344585776329, + 0.02246353216469288, + 0.05694645270705223, + 0.04595059156417847, + -0.0388513058423996, + 0.03461457043886185, + -0.0022867487277835608, + -0.048993054777383804, + -0.023689327761530876, + -0.07536367326974869, + -0.011515623889863491, + 0.007936140522360802, + -0.03663696348667145, + -0.06312127411365509, + -0.04278803989291191, + 0.03572988882660866, + -0.02872752957046032, + -0.02007090300321579, + -0.01753923110663891, + -0.044812753796577454, + -0.025302119553089142, + 0.03735737130045891, + -0.03653990104794502, + 0.011053361929953098, + -0.04280798137187958, + -0.008700693026185036, + 0.031340040266513824, + 0.0599483847618103, + -0.0002709626860450953, + -0.0010187489679083228, + -0.008950572460889816, + -0.017538035288453102, + -0.009258774109184742, + 0.0011439865920692682, + -0.057143427431583405, + -0.04661299288272858, + -0.014264841563999653, + 0.0020582641009241343, + -0.013787128031253815, + -0.012294617481529713, + 0.04054946079850197, + -0.05668805539608002, + -0.04687212407588959, + 0.024867137894034386, + -0.029429618269205093, + 0.039136938750743866, + -0.0010047232499346137, + 0.008986036293208599, + -0.034250419586896896, + 0.021724063903093338, + 0.019622910767793655, + -0.06410157680511475, + -0.0016147876158356667, + -0.0038255443796515465, + 0.06889861822128296, + -0.06239516660571098, + -0.02361162379384041, + 0.019185150042176247, + -0.011949440464377403, + -0.027609966695308685, + -0.01281854696571827, + 0.000356674863724038, + -0.00711772870272398, + -0.09345481544733047, + -0.00025708225439302623, + -0.006874196697026491, + -0.02692345716059208, + -0.06142395734786987, + -0.022152185440063477, + 0.029652997851371765, + 0.014645223505795002, + 0.047659821808338165, + 0.0022297888062894344, + -0.07625388354063034, + 0.0077497707679867744, + -0.04298599436879158, + -0.01545044593513012, + -0.008620177395641804, + 0.03281245008111, + 0.04964756593108177, + 0.031187480315566063, + 0.02064322493970394, + -0.005366362631320953, + -0.042736537754535675, + 0.05334664508700371, + -0.021536031737923622, + 0.012270762585103512, + -0.080686554312706, + -0.002283904468640685, + -0.0699366107583046, + -0.010594785213470459, + -0.04290773347020149, + 0.002180918585509062, + -0.0506034679710865, + -0.004024667665362358, + -0.00997102353721857, + -0.04159633070230484, + -0.027407394722104073, + 0.06402017176151276, + -0.022402895614504814, + -0.03860772028565407, + -0.009718614630401134, + -0.011002982966601849, + -0.022690828889608383, + 0.008263494819402695, + -0.030103253200650215, + 0.029220368713140488, + 0.01350152213126421, + -0.046072036027908325, + 0.08582505583763123, + 0.013142880983650684, + 0.04821730777621269, + 0.004140455741435289, + 0.0662679374217987, + -0.0016365874325856566, + -0.0327213853597641, + -0.06150257959961891, + -0.027527324855327606, + 0.0035866505932062864 + ], + "negated_query": [ + 0.08803019672632217, + 0.01441983412951231, + -0.12369474768638611, + -0.006890435237437487, + 0.0586794912815094, + -0.012951640412211418, + 0.03609541058540344, + 0.029055267572402954, + 0.05665610730648041, + 0.002597457729279995, + -0.012526022270321846, + 0.06433925777673721, + 0.03525780886411667, + 0.02601497806608677, + -0.045713700354099274, + -0.006677167024463415, + 0.015442367643117905, + -0.10731576383113861, + 0.015808356925845146, + 0.052059877663850784, + -0.037920814007520676, + -0.02717948891222477, + -0.027718955650925636, + 0.01080583781003952, + 0.019329039379954338, + -0.0370841920375824, + 0.06882450729608536, + -0.010026969946920872, + -0.006460671313107014, + 0.04038366302847862, + 0.07035281509160995, + 0.052522122859954834, + 0.032837800681591034, + -0.024208994582295418, + -0.06300845742225647, + -0.03750999644398689, + 0.05014652758836746, + 0.020283525809645653, + -0.016282085329294205, + -0.03311882168054581, + 0.05887452885508537, + 0.02935974858701229, + 0.032644275575876236, + -0.00672399764880538, + -0.02717270515859127, + 0.02793193608522415, + 0.057740382850170135, + -0.04075304791331291, + 0.03400949761271477, + -0.02745135873556137, + 0.04541577026247978, + -0.022252993658185005, + -0.011865317821502686, + -0.00972041580826044, + -0.01957753114402294, + 0.011410812847316265, + 0.01980406977236271, + 0.0145584587007761, + 0.019224734976887703, + -0.054445985704660416, + 0.08363769948482513, + 0.005364919081330299, + 0.039745863527059555, + 0.07198711484670639, + 0.015904143452644348, + -0.0160105861723423, + -0.08471553772687912, + 0.02083435095846653, + 0.012888353317975998, + -0.032195597887039185, + 0.06124695762991905, + 0.06137024983763695, + -0.0024819448590278625, + 0.013849901035428047, + 0.015541641041636467, + 0.027382301166653633, + -0.04800468683242798, + -0.026158928871154785, + 0.054854944348335266, + 0.006685914006084204, + 0.016809506341814995, + -0.02281312085688114, + 0.06223977729678154, + -0.01214031595736742, + 0.028964269906282425, + 0.0030311758164316416, + -0.017352329567074776, + -0.03007711097598076, + 0.028741618618369102, + 0.07847274839878082, + 0.01699373498558998, + 0.007976025342941284, + 0.07832422107458115, + 0.05558042973279953, + -0.010984061285853386, + 0.03360927104949951, + -0.008808249607682228, + 0.04348234087228775, + 0.015944112092256546, + -0.06517495959997177, + 0.004715932998806238, + -0.06212177127599716, + -0.006052685435861349, + 0.015178618021309376, + 0.02072238363325596, + 0.017017370089888573, + 0.0031142006628215313, + 0.006121623329818249, + -0.0007535610347986221, + -0.014271221123635769, + -0.058177974075078964, + 0.03232840821146965, + -0.02632732130587101, + -0.0017654814291745424, + 0.014823820441961288, + -0.017708659172058105, + 0.07806055247783661, + -0.04045442119240761, + -0.005915777292102575, + -0.001936405780725181, + 0.004172272980213165, + 0.005228932946920395, + 0.03470851108431816, + 0.04214303195476532, + 0.034868672490119934, + -0.051453977823257446, + -0.023936092853546143, + 0.014487066306173801, + 0.034290965646505356, + -0.07711129635572433, + -0.04324188828468323, + 0.03222312405705452, + -0.06146906688809395, + 0.06686235964298248, + 0.01642962358891964, + 0.043170567601919174, + 0.024536799639463425, + -0.04832347109913826, + 0.013638394884765148, + -0.00643573421984911, + 0.014131726697087288, + 0.06847158819437027, + -0.027357716113328934, + -0.022445309907197952, + 0.022324630990624428, + -0.05437299236655235, + 0.053629204630851746, + -0.018233079463243484, + 0.027034876868128777, + 0.024065600708127022, + -0.0031720208935439587, + -0.023442402482032776, + -0.022341443225741386, + 0.00251897145062685, + 0.00719880498945713, + -0.03614075109362602, + -0.025827324017882347, + -0.0379357747733593, + 0.06621306389570236, + 0.0314473882317543, + 0.057292141020298004, + 0.03099074773490429, + -0.03516566753387451, + 0.05331430211663246, + 0.0299970842897892, + -0.029804078862071037, + -0.006949882488697767, + 0.006590934470295906, + 0.03111850470304489, + 0.009300011210143566, + -0.08359431475400925, + -0.003680884838104248, + -0.005156670697033405, + -0.013560909777879715, + 0.010861920192837715, + 0.0016015033470466733, + 0.017965268343687057, + -0.01048897672444582, + 0.0034879918675869703, + -0.045929327607154846, + 0.0008584904717281461, + -0.005336532834917307, + 0.03719300031661987, + 0.07092759758234024, + 0.03354746475815773, + -0.01084791962057352, + 0.016288822516798973, + 0.044132113456726074, + -0.01831003837287426, + 0.004779826384037733, + 0.007770434953272343, + 0.002813217928633094, + -0.052447278052568436, + -0.023685190826654434, + -0.03955240920186043, + 0.0037335706874728203, + 0.03193110600113869, + 0.02538239024579525, + 0.052729275077581406, + 0.011795993894338608, + -0.062401626259088516, + -0.025446705520153046, + -0.030362918972969055, + 0.024961940944194794, + -0.06499286741018295, + 0.022669576108455658, + 0.02097843773663044, + 0.03499065339565277, + -0.03168269991874695, + -0.024327313527464867, + 0.04741762951016426, + -0.034746475517749786, + -0.02116261050105095, + -0.007397667970508337, + 0.021244607865810394, + -0.03595792129635811, + -0.003207304747775197, + -0.028044579550623894, + -0.030822744593024254, + 0.037538573145866394, + 0.0125344917178154, + -0.03017033264040947, + -0.0014150053029879928, + -0.01554068736732006, + -0.028402291238307953, + -0.03272860124707222, + -0.059385523200035095, + -0.03773213177919388, + -0.060035355389118195, + -0.05537812411785126, + 0.008742243982851505, + -0.07325819134712219, + 0.07683634757995605, + 0.022092506289482117, + 0.0013929421547800303, + 0.019228562712669373, + -0.008182263001799583, + 0.03717733919620514, + 0.012803268618881702, + 0.040656767785549164, + -0.00496508926153183, + 0.028195485472679138, + -0.011157299391925335, + 0.0037855573464185, + 0.007505566347390413, + -0.010570373386144638, + -0.028106266632676125, + -0.06738611310720444, + 0.029769349843263626, + 0.024327784776687622, + -0.05975712090730667, + 0.012055479921400547, + 0.021029533818364143, + 0.04124658554792404, + -0.004652858711779118, + -0.02064531296491623, + -0.03308633714914322, + -0.0036878047976642847, + -0.012551510706543922, + -0.00900215469300747, + 0.012926025316119194, + 0.0075853243470191956, + -0.002057128818705678, + -0.041860878467559814, + -0.014323210343718529, + -0.040527526289224625, + -0.03725213184952736, + -0.03507467359304428, + 0.08040640503168106, + -0.002086549997329712, + 0.04381782189011574, + 0.00839106272906065, + 0.010020226240158081, + 0.08794788271188736, + 0.015523101203143597, + 0.009381558746099472, + 0.014189708977937698, + 0.032114818692207336, + 0.002737700007855892, + 0.07584260404109955, + 0.012314578518271446, + -0.019695337861776352, + -0.08122554421424866, + 0.027850378304719925, + 0.03221336007118225, + 0.03824809193611145, + 0.019792107865214348, + -0.014485212042927742, + 0.003672859398648143, + 0.042179547250270844, + 0.04454492777585983, + 0.07153574377298355, + -0.015526884235441685, + 0.05917206406593323, + 0.007853880524635315, + 0.010212859138846397, + 0.02243076078593731, + 0.051636047661304474, + 0.026406290009617805, + -0.02135828137397766, + 0.04781036823987961, + -2.577078339527361e-05, + 0.0430721677839756, + 0.06617778539657593, + -0.0020474432967603207, + -0.015059993602335453, + 0.034203752875328064, + 0.023967260494828224, + 0.02603805996477604, + -0.06550758332014084, + -0.04494963213801384, + 0.011719988659024239, + -0.05385105684399605, + 0.05502946302294731, + -0.043605636805295944, + 0.06830824166536331, + 0.03998614847660065, + 0.008196220733225346, + 0.026976149529218674, + 0.0020626429468393326, + -0.011028044857084751, + -0.011211400851607323, + -0.015026313252747059, + -0.034829460084438324, + 0.03771417960524559, + 0.016299428418278694, + -0.002501686569303274, + 0.034092977643013, + -0.04472998529672623, + -0.006408214569091797, + -0.0014052799670025706, + 0.06803295016288757, + 0.034980159252882004, + -0.039513684809207916, + -0.04194539785385132, + -0.013605637475848198, + -0.0005803612293675542, + 0.004862042143940926, + -0.024970071390271187, + 0.02600487694144249, + 0.09968989342451096, + -0.03869671747088432, + 0.012345054186880589, + -0.008065305650234222, + 0.017035799100995064, + 0.006094327662140131, + -0.08198249340057373, + -0.03454209119081497, + 0.042327020317316055, + -0.007063239347189665, + -0.005699435714632273, + 0.037089284509420395, + 0.021873552352190018, + -0.03242116421461105, + -0.004578231833875179, + -0.028576500713825226, + 0.015420480631291866, + 0.029331287369132042, + -0.025237509980797768, + -0.012465028092265129, + 0.045775070786476135, + -0.002410859102383256, + -0.028766656294465065, + -0.003961813170462847, + 0.040774233639240265, + 0.08639143407344818, + 0.02851099707186222, + 0.007490535266697407, + 0.023477498441934586, + -0.014866622164845467, + 0.028769122436642647, + -0.03047286905348301, + 9.772604244062677e-05, + 0.005880428943783045, + 0.008305522613227367, + 0.006700580939650536, + 0.008410410024225712, + -0.0036310588475316763, + -0.012012984603643417, + 0.061623286455869675, + 0.05306493863463402, + -0.02555815875530243, + 0.05487783998250961, + 0.004268114920705557, + -0.03856393322348595, + -0.03729330748319626, + 0.010395441204309464, + -0.011382927186787128, + -0.018894465640187263, + 0.02271888218820095, + -0.008624637499451637, + -0.030721401795744896, + -0.022017747163772583, + 0.01239986065775156, + 0.027090521529316902, + -0.04611373692750931, + 0.05257301777601242, + -0.006174738053232431, + 0.0043242922984063625, + 0.013128358870744705, + -0.07540897279977798, + -0.023426584899425507, + -0.024870101362466812, + -0.00913197360932827, + -0.027949854731559753, + 0.030062522739171982, + -0.04290373995900154, + -0.0674513429403305, + 0.047676946967840195, + -0.010956048034131527, + -0.00693393824622035, + 0.021506665274500847, + 0.021503182128071785, + 0.0147182522341609, + -0.019055025652050972, + 0.010448066517710686, + 0.0005079724942333996, + -0.004043302033096552, + -0.009762043133378029, + -0.029867447912693024, + 0.02215317264199257, + 0.035062890499830246, + 0.016776062548160553, + 0.00935530848801136, + -0.036127299070358276, + 0.03171919286251068, + -0.0011559673584997654, + -0.026294251903891563, + -0.05362481251358986, + -0.09523624181747437, + 0.03895895183086395, + 0.0052637760527431965, + 0.06470480561256409, + -0.012449625879526138, + -0.06353572010993958, + 0.012496470473706722, + 0.00942989718168974, + 0.047553326934576035, + -0.043613724410533905, + 0.05022728815674782, + -0.017217053100466728, + -0.033371418714523315, + 0.03919738158583641, + -0.012712509371340275, + -0.010419762693345547, + 0.10398604720830917, + 0.02439045161008835, + -0.02539542131125927, + -0.016881907358765602, + 0.07965626567602158, + 0.016225993633270264, + 0.06458791345357895, + 0.01233601476997137, + 0.008848719298839569, + 0.040648218244314194, + -0.031145447865128517, + -0.005041076336055994, + 0.03390458971261978, + 0.00620918208733201, + 0.02936151809990406, + 0.05060628429055214, + 0.025626283138990402, + -0.011437000706791878, + 0.02888123132288456, + -0.014495858922600746, + -0.028653360903263092, + 0.007882745936512947, + -0.0005044366116635501, + 0.03287528455257416, + 0.0662863552570343, + -0.06724715232849121, + 0.010770879685878754, + 0.011163182556629181, + -0.001956851687282324, + -0.03869105502963066, + 0.04745686799287796, + -0.004909251816570759, + 0.020520001649856567, + 0.010636516846716404, + 0.06486985087394714, + 0.008955512195825577, + 0.0012331890175119042, + -0.0015953737311065197, + -0.05135318636894226, + 0.014675813727080822, + 0.031467653810977936, + 0.055022791028022766, + -0.04686162248253822, + 0.02624928019940853, + 0.04586183652281761, + -0.021684804931282997, + -0.010138409212231636, + 0.016854969784617424, + -0.012121528387069702, + -0.08197088539600372, + -0.05958136171102524, + -0.046961743384599686, + 0.026981301605701447, + -0.04192891716957092, + 0.007830357179045677, + -0.03587086498737335, + 0.02673325128853321, + 0.01451930496841669, + -0.007140323985368013, + -0.016428299248218536, + -0.008013150654733181, + 0.02398492395877838, + -0.07893500477075577, + -0.0027793466579169035, + 0.009952414780855179, + -0.00583701953291893, + -0.031366344541311264, + 0.01571049727499485, + 0.0399349071085453, + 0.01130961999297142, + -0.042996495962142944, + 0.03920019418001175, + 0.01689073070883751, + -0.06513823568820953, + 0.004528265446424484, + -0.03200265020132065, + 0.02613486349582672, + 0.020696226507425308, + 0.013190769590437412, + -0.0657515674829483, + -0.007943403907120228, + 0.006468652281910181, + -0.010789946652948856, + 0.0038234046660363674, + -0.011800720356404781, + 0.013830783776938915, + 0.06282223761081696, + -0.02622964233160019, + -0.015412404201924801, + -0.011969777755439281, + -0.028739698231220245, + -0.04248332977294922, + 0.006648917682468891, + 0.033846769481897354, + 0.004392351023852825, + -0.011494670063257217, + 0.03448839485645294, + -0.026833750307559967, + 0.011477788910269737, + -0.026624618098139763, + -0.013929652981460094, + -0.009020065888762474, + -0.010551544837653637, + 0.004251671023666859, + -0.07565367221832275, + 0.03725288063287735, + -0.06876791268587112, + 0.009487212635576725, + -0.026552995666861534, + 0.02256244234740734, + -0.0751008465886116, + 0.04016231372952461, + -0.014138401485979557, + -0.03341599181294441, + -0.06533192098140717, + -0.04048802703619003, + -0.006438093725591898, + 0.04065500572323799, + 0.004079723730683327, + 0.05583799257874489, + -0.010733210481703281, + -0.02130304090678692, + -0.0010593571932986379, + -0.04459553584456444, + 0.022099360823631287, + -0.018185749650001526, + -0.032591234892606735, + -0.04763538017868996, + -0.03345683589577675, + -0.031029051169753075, + -0.011011422611773014, + -0.008001141250133514, + -0.016032151877880096, + -0.011919134296476841, + -0.05267670005559921, + 0.003362351329997182, + -0.0346994511783123, + 0.0625842809677124, + 0.005930026993155479, + -0.0024445534218102694, + -0.05186312645673752, + 0.004912829026579857, + -0.03736477717757225, + 0.027696993201971054, + -0.026822414249181747, + 0.005218078847974539, + -0.006278188433498144, + -0.0194089412689209, + -0.007091018836945295, + 0.03947738930583, + -0.028066620230674744, + -0.03431905433535576, + -0.052888255566358566, + 0.011450245976448059, + -0.0357879102230072, + 0.047211695462465286, + -0.018787266686558723, + -0.0049520074389874935, + -0.016120577231049538, + 0.021601272746920586, + 0.06714802235364914, + 0.03662010654807091, + 0.032859109342098236, + -0.018206261098384857, + -0.07728938013315201, + 0.03259894624352455, + 0.001409130753017962, + -0.00913365837186575, + -0.0179558377712965, + 0.05472410097718239, + -0.056782905012369156, + 0.04299485683441162, + -0.004109925124794245, + 0.0009879101999104023, + -0.014512471854686737, + 0.0008622151799499989, + -0.04418473318219185, + 0.07803704589605331, + 0.016182271763682365, + 0.02109469287097454, + 0.003243178129196167, + -0.01709725148975849, + -0.07664024829864502, + -0.012772542424499989, + 0.03514208644628525, + -0.016776112839579582, + 0.05037075653672218, + -0.04638507589697838, + -0.06973011046648026, + -0.06624900549650192, + 0.05793401971459389, + 0.02124779485166073, + 0.018780341371893883, + -0.02122405730187893, + -0.031974852085113525, + 0.02899823896586895, + 0.021247627213597298, + 0.021564919501543045, + -0.0030831568874418736, + -0.05233597382903099, + 0.0011642068857327104, + 0.010012886486947536, + 0.048615097999572754, + 0.03160793334245682, + -0.030557643622159958, + 0.02715899795293808, + 0.05047673359513283, + 0.033183272927999496, + -0.04027437046170235, + 0.028374742716550827, + -0.009564025327563286, + -0.04109463468194008, + -0.015534540638327599, + -0.07033000886440277, + -0.012822859920561314, + -0.00021713356545660645, + -0.026955224573612213, + -0.06685400009155273, + -0.030269447714090347, + 0.044084224849939346, + -0.034719318151474, + -0.03879636153578758, + -0.01164296269416809, + -0.05500242859125137, + -0.014473581686615944, + 0.044786252081394196, + -0.035995274782180786, + 0.006783656310290098, + -0.03805335983633995, + 0.009420794434845448, + 0.03579188138246536, + 0.07919053733348846, + -0.0011867779539898038, + 0.005728098098188639, + -0.013946125283837318, + -0.01719212904572487, + 0.004764148965477943, + 0.008631885051727295, + -0.07042191177606583, + -0.046211522072553635, + -0.007018572650849819, + 0.0049719009548425674, + -0.020748848095536232, + -0.009059038944542408, + 0.03201454132795334, + -0.04523981735110283, + -0.04867248982191086, + 0.03139074146747589, + -0.037596430629491806, + 0.04073108360171318, + -0.008252826519310474, + 0.00416927132755518, + -0.046306099742650986, + 0.008201162330806255, + 0.009999132715165615, + -0.0568917915225029, + 0.0010095733450725675, + 0.0024690062273293734, + 0.05081987380981445, + -0.05488235130906105, + -0.018779734149575233, + 0.021062249317765236, + -0.005111983045935631, + -0.030285440385341644, + -0.015109477564692497, + -0.0006127436063252389, + -0.001149228191934526, + -0.09195038676261902, + 0.012088808231055737, + -0.011120999231934547, + -0.03583262860774994, + -0.059735555201768875, + -0.026356831192970276, + 0.027491403743624687, + 0.01326157245784998, + 0.0344899445772171, + -0.01657005213201046, + -0.07343795895576477, + -0.0012000398710370064, + -0.03622210770845413, + -0.027086276561021805, + 0.0009199509513564408, + 0.02451152168214321, + 0.04533359035849571, + 0.026923680678009987, + 0.005505596287548542, + -0.004136490169912577, + -0.04326382651925087, + 0.0559898316860199, + -0.027453003451228142, + 0.009812010452151299, + -0.0924718901515007, + -0.004833732265979052, + -0.06920479238033295, + -0.012676259502768517, + -0.03018173575401306, + -0.0036493882071226835, + -0.046804916113615036, + 0.0035351519472897053, + -0.006282047834247351, + -0.030421141535043716, + -0.02224845252931118, + 0.05186721310019493, + -0.028449509292840958, + -0.032835159450769424, + -0.009203864261507988, + -0.0007472375873476267, + -0.036180514842271805, + 0.01365070603787899, + -0.028763802722096443, + 0.023995568975806236, + 0.022189777344465256, + -0.050793807953596115, + 0.09840061515569687, + 0.019764447584748268, + 0.04769085347652435, + 0.0012144512729719281, + 0.05996841564774513, + 0.007713211700320244, + -0.03686460480093956, + -0.06426706910133362, + -0.013657934963703156, + -0.014375492930412292 + ], + "paraphrase_stored": [ + 0.08559243381023407, + 0.009842374362051487, + -0.14875386655330658, + -0.011246277019381523, + 0.03445244953036308, + 0.0025363238528370857, + 0.0014101519482210279, + 0.03712672367691994, + 0.06504547595977783, + 0.059952206909656525, + -0.052484180778265, + 0.060048557817935944, + 0.020804474130272865, + -0.011023890227079391, + 0.02452288381755352, + 0.016849294304847717, + 0.0062898751348257065, + -0.09507152438163757, + -0.05022191256284714, + -0.004747831262648106, + 0.07766755670309067, + -0.022368313744664192, + -0.0546838641166687, + 0.036171503365039825, + 0.035223737359046936, + -0.009356433525681496, + 0.040863338857889175, + -0.028292350471019745, + 0.01360828522592783, + 0.011856826953589916, + 0.0547950454056263, + 0.030047668144106865, + 0.025138532742857933, + 0.0005798496422357857, + 0.0034215827472507954, + -0.058507759124040604, + 0.018415819853544235, + 0.009610574692487717, + 0.01226664800196886, + -0.028091199696063995, + 0.025466619059443474, + 0.04785259813070297, + 0.03547630459070206, + -0.056495580822229385, + 0.05355248972773552, + 0.04657472297549248, + 0.0841885358095169, + -0.014562705531716347, + 0.049848463386297226, + -0.007924485020339489, + 0.045994751155376434, + -0.05327276512980461, + -0.023161152377724648, + 0.003762393957003951, + 0.03855687379837036, + 0.028928380459547043, + 0.034257449209690094, + -0.02459827810525894, + 0.02482028119266033, + -0.06988734006881714, + 0.055181436240673065, + 0.036535270512104034, + 0.006629236973822117, + 0.09136529266834259, + -0.005317928269505501, + -0.006567250471562147, + -0.01931305043399334, + 0.04032023251056671, + 0.06025323644280434, + -0.05908028408885002, + 0.03351153805851936, + 0.07157720625400543, + -0.005289190914481878, + -0.011414002627134323, + 0.029123596847057343, + 0.060280993580818176, + -0.012461308389902115, + -0.016583716496825218, + 0.04080566391348839, + 0.03917795792222023, + -0.00022497509780805558, + -0.018208157271146774, + 0.10318076610565186, + -0.013216572813689709, + 0.08709350973367691, + 0.01581662893295288, + 0.007153725251555443, + 0.014002084732055664, + 0.0024488698691129684, + 0.07610637694597244, + -0.005151715595275164, + -0.0034706457518041134, + 0.0441545769572258, + 0.016411518678069115, + -0.006700129248201847, + -0.020791249349713326, + -0.03811497241258621, + 0.03132147341966629, + -0.05703050270676613, + 0.036725953221321106, + -0.0354965403676033, + -0.09758485108613968, + 0.01399222295731306, + -0.02024834230542183, + 0.04574038088321686, + 0.021420855075120926, + 0.015847191214561462, + -0.0005580008728429675, + 0.024298986420035362, + 0.012036430649459362, + 0.00430859113112092, + 0.014498040080070496, + -0.010937890037894249, + 0.012781128287315369, + -0.021216100081801414, + -0.019705291837453842, + 0.06084243580698967, + -0.014114347286522388, + 0.008670617826282978, + 0.015327435918152332, + -0.007745802868157625, + -0.012738204561173916, + 0.014963758178055286, + 0.01306045800447464, + -0.023924916982650757, + 0.00024695563479326665, + 0.00769945839419961, + 0.024030283093452454, + 0.06081337109208107, + -0.02536182850599289, + -0.005305880680680275, + 0.021629048511385918, + -0.005834938958287239, + 0.08064417541027069, + 0.016470177099108696, + 0.044307783246040344, + -0.006071437615901232, + -0.03544909134507179, + -0.023147746920585632, + -0.017029820010066032, + 0.015229497104883194, + 0.02388908714056015, + 0.037660036236047745, + -0.020535416901111603, + -0.029628312215209007, + -0.02199641242623329, + 0.053438831120729446, + -0.0190187469124794, + -0.002679127501323819, + 0.04507151618599892, + 0.014501238241791725, + -0.012175007723271847, + -0.0007543463725596666, + 0.01791328191757202, + 0.03290370851755142, + -0.020160146057605743, + -0.04009569063782692, + -0.01580999791622162, + 0.050243522971868515, + 0.032233916223049164, + 0.06523972004652023, + 0.02686295658349991, + -0.05184827744960785, + 0.07882142812013626, + -0.024837715551257133, + -0.024337904527783394, + 0.0055742086842656136, + 0.04089110717177391, + 0.028029384091496468, + -0.0375794880092144, + -0.026770973578095436, + -0.02880687639117241, + -0.04156867042183876, + 0.01836942508816719, + 0.04283152520656586, + 0.030804844573140144, + 0.0005352571606636047, + 0.005757465958595276, + 0.013506046496331692, + -0.005427876487374306, + 0.007215488702058792, + -0.04989509657025337, + -0.009543200954794884, + 0.07203084975481033, + 0.006215846166014671, + -0.026240482926368713, + 0.024101631715893745, + 0.007456653751432896, + -0.03135102987289429, + -0.023157574236392975, + -0.012562423944473267, + 0.03105107694864273, + -0.045698557049036026, + 0.012805931270122528, + -0.01784401945769787, + -0.022379428148269653, + 0.0049925618804991245, + 0.04749347269535065, + 0.03218671679496765, + 0.006697634235024452, + -0.035155534744262695, + -0.02194199338555336, + -0.03165886551141739, + -0.013961905613541603, + -0.0690598413348198, + 0.04232116788625717, + 0.014831597916781902, + -0.038370028138160706, + -0.04627394676208496, + -0.03151627257466316, + 0.06606084108352661, + -0.00873227883130312, + -0.0379280149936676, + -0.012215091846883297, + -0.005583822727203369, + -0.043724581599235535, + -0.02403172478079796, + -0.026217250153422356, + -0.02290301024913788, + -0.011960485018789768, + 0.01231803372502327, + 0.022784298285841942, + 0.007789182011038065, + -0.053888801485300064, + -0.04507102072238922, + -0.029718169942498207, + -0.03972852602601051, + -0.032666757702827454, + -0.056811507791280746, + -0.00867325346916914, + -0.01860269159078598, + -0.07960496097803116, + 0.057835645973682404, + -0.030115433037281036, + -0.013475635088980198, + 0.022758906707167625, + -0.0037202455569058657, + 0.05477618798613548, + -0.006703744642436504, + 0.018015628680586815, + -0.057075899094343185, + 0.03497724235057831, + 0.010888762772083282, + -0.021553363651037216, + -0.035903945565223694, + -0.004853937309235334, + -0.011506645940244198, + -0.04147568345069885, + 0.011851251125335693, + 0.03635961189866066, + 0.020410215482115746, + 0.012685278430581093, + 0.02373061515390873, + 0.023100029677152634, + 0.053562551736831665, + 0.01660570688545704, + -0.032669685781002045, + 0.0038370213005691767, + -0.006952316500246525, + -0.029088599607348442, + 0.01725170761346817, + 0.016899457201361656, + 0.018048638477921486, + -0.015196208842098713, + 0.005054839886724949, + -0.08002091199159622, + -0.04584382474422455, + -0.00042423384729772806, + 0.044183485209941864, + 0.008920996449887753, + -0.0409565344452858, + 0.02773190848529339, + -0.0037370361387729645, + 0.05941489338874817, + -0.021077407523989677, + 0.025148987770080566, + 0.021954946219921112, + 0.004559503868222237, + -0.015824608504772186, + 0.040334552526474, + -0.008979825302958488, + -0.013821491040289402, + -0.06590146571397781, + -0.02446400187909603, + -0.014483576640486717, + -0.025752509012818336, + 0.010322700254619122, + -0.025415753945708275, + -0.014075323939323425, + -0.03556903079152107, + 0.026701537892222404, + 0.03370651602745056, + 0.013759996742010117, + 0.019859718158841133, + 0.0016594273038208485, + 0.0731230080127716, + 0.0007492731674574316, + 0.03268574923276901, + 0.0039062658324837685, + -0.004929174669086933, + 0.052208997309207916, + -0.009337276220321655, + 0.015680907294154167, + 0.035059764981269836, + -0.025274161249399185, + -0.031218087300658226, + 0.020477237179875374, + -0.007020899094641209, + 0.014568591490387917, + -0.026472577825188637, + -0.0638972669839859, + -0.02650357410311699, + -0.07444790750741959, + 0.06912273168563843, + -0.05845781788229942, + 0.06371675431728363, + 0.08392103016376495, + -0.00614855857565999, + 0.03413603827357292, + 0.0165393128991127, + -0.01577025279402733, + 0.0381087064743042, + -0.007321950979530811, + 1.6589287042734213e-05, + 0.021559428423643112, + -0.010127834044396877, + -0.00698620080947876, + 0.03259173035621643, + -0.02829882688820362, + -0.028951870277523994, + 0.003238728502765298, + 0.04564371332526207, + 0.01979057677090168, + -0.04044875502586365, + -0.0335860438644886, + 0.017907796427607536, + 0.00994102656841278, + -0.010733632370829582, + -0.03871041163802147, + 0.05380979925394058, + 0.1259676069021225, + -0.0482681468129158, + 0.045811038464307785, + -0.046801093965768814, + 0.016071954742074013, + 0.03994050621986389, + -0.01180534902960062, + -0.015636157244443893, + 0.06010206788778305, + 0.01109381578862667, + -0.017879558727145195, + -0.009153347462415695, + 0.005668624769896269, + -0.03346691280603409, + 0.003949945326894522, + -0.03422941640019417, + 0.028608161956071854, + 0.06813468039035797, + -0.016967395320534706, + -0.019231421872973442, + 0.04374415799975395, + -0.0025859603192657232, + -0.01805534027516842, + -0.006802863907068968, + -0.006616140715777874, + 0.01628243178129196, + -0.003234961535781622, + -0.009700745344161987, + 0.02696816436946392, + 0.03346114233136177, + 0.03968378156423569, + -0.01109296828508377, + 0.03727065399289131, + 0.017295608296990395, + -0.029650039970874786, + 0.008499293588101864, + -0.045377254486083984, + 0.012166783213615417, + -0.011831352487206459, + 0.07006421685218811, + 0.04310472682118416, + 0.012414614669978619, + 0.020772185176610947, + -0.009970481507480145, + -0.05212648585438728, + -0.006524672731757164, + 0.023321976885199547, + 0.0035891460720449686, + -0.017744481563568115, + 0.032371312379837036, + -0.006925185676664114, + -0.025137901306152344, + -0.015082310885190964, + -0.011952186934649944, + 0.037765465676784515, + -0.06326253712177277, + -0.020755892619490623, + -0.031475652009248734, + 0.03263571485877037, + -0.028295086696743965, + -0.009050080552697182, + 0.0028418367728590965, + 0.00931322667747736, + 0.014593523927032948, + -0.05318643897771835, + 0.057505201548337936, + -0.006058127619326115, + -0.06478461623191833, + 0.0289616696536541, + 0.01827518828213215, + 0.017006775364279747, + 0.02856292761862278, + -0.03230912610888481, + 0.005105618853121996, + 0.01909705065190792, + 0.028320705518126488, + -0.01643409952521324, + 0.06910761445760727, + -0.003952719271183014, + 0.007183264009654522, + 0.03584799915552139, + 0.005590795539319515, + 0.03974426910281181, + 0.010010805912315845, + 0.005192843731492758, + 0.05257198587059975, + -0.0167266596108675, + -0.008139639161527157, + -0.009892238304018974, + -0.10205423086881638, + 0.00937082339078188, + 0.03015679307281971, + 0.031140141189098358, + -0.006637824233621359, + -0.04403674602508545, + -0.01563630998134613, + 0.022618064656853676, + 0.0273640975356102, + 0.018943974748253822, + 0.09648949652910233, + -0.00935632735490799, + -0.0427912212908268, + -0.0012795112561434507, + 0.017038073390722275, + 0.030652450397610664, + 0.09883879125118256, + 0.020626559853553772, + -0.01788569986820221, + -0.033801302313804626, + 0.023587845265865326, + -0.01217307522892952, + 0.0060767619870603085, + 0.003070375882089138, + 0.011284979991614819, + 0.03689638525247574, + -0.009472325444221497, + -0.02210124023258686, + 0.039725542068481445, + 0.007769444026052952, + 0.07063303142786026, + 0.07664623856544495, + 0.012620355002582073, + 0.023771464824676514, + 0.042755767703056335, + -0.026695095002651215, + -0.0026501028332859278, + 0.007798066362738609, + -0.048368748277425766, + -0.015821000561118126, + 0.09781331568956375, + -0.025890177115797997, + -0.016697678714990616, + -0.014842016622424126, + 0.030325304716825485, + -0.016888469457626343, + -0.02907811664044857, + -0.0375995934009552, + -0.0176241435110569, + 0.00805443711578846, + 0.05711852014064789, + -0.02070309966802597, + -0.010950645431876183, + 0.011774285696446896, + -0.03972206637263298, + 0.012846319936215878, + 0.059013381600379944, + 0.03199749439954758, + -0.007315813563764095, + 0.035526275634765625, + 0.05261792615056038, + -0.02831386961042881, + 0.03746639937162399, + 0.0012152529088780284, + -0.042228247970342636, + 0.004973540082573891, + -0.042253635823726654, + -0.05909346789121628, + 0.06227242574095726, + 0.011853918433189392, + -0.0160143431276083, + 0.01968584954738617, + 0.07765201479196548, + -0.04083618149161339, + -0.021825438365340233, + 0.009964553639292717, + -0.042228322476148605, + 0.008212590590119362, + -0.03077121078968048, + -0.055946122854948044, + 0.009350646287202835, + -0.026209115982055664, + 0.032271403819322586, + 0.013050264678895473, + 0.01578144170343876, + 0.03232898563146591, + -0.019732650369405746, + -0.02766835130751133, + -0.012597980909049511, + -0.04931832104921341, + 0.013591364026069641, + 0.008107362315058708, + 0.015708569437265396, + -0.02180664800107479, + -0.04619787260890007, + -0.09399691969156265, + -0.047207675874233246, + -0.011559261940419674, + -0.040124163031578064, + 0.02089710719883442, + 0.028928076848387718, + 0.03876453638076782, + 0.0006974693387746811, + -0.05577893182635307, + 0.013372691348195076, + -0.0006633849116042256, + 0.008149407804012299, + -0.05586523562669754, + -0.015462381765246391, + -0.007323152851313353, + 0.025721849873661995, + -0.007852497510612011, + -0.022624626755714417, + -0.01967291161417961, + 0.02013041079044342, + -0.03823531046509743, + -0.025752747431397438, + -0.01556487288326025, + 0.009781177155673504, + 0.021418839693069458, + -0.051429811865091324, + 0.0015191597631201148, + -0.06011909246444702, + -0.024669991806149483, + -0.05140792578458786, + -0.004898873623460531, + -0.04558511823415756, + -0.006597441621124744, + -0.010003635659813881, + 0.0036268525291234255, + -0.05564330518245697, + 0.025645332410931587, + 0.024786967784166336, + 0.034006476402282715, + 0.0030172315891832113, + 0.00039365989505313337, + 0.018077494576573372, + -0.009391117841005325, + -0.03562731295824051, + 0.011890481226146221, + 0.0052057537250220776, + 0.05256623402237892, + -0.02976665087044239, + -0.03810829296708107, + -0.012604281306266785, + -0.02506493218243122, + 0.016687670722603798, + -0.004299490712583065, + -0.03658037260174751, + -0.008785178884863853, + -0.04062521457672119, + -0.025814466178417206, + 0.0032796296291053295, + 0.027658900246024132, + -0.0078123495914042, + 0.010806547477841377, + -0.05228220671415329, + -0.0014937857631593943, + -0.046340201050043106, + 0.06664662808179855, + 0.0020375445019453764, + -0.036938395351171494, + -0.0001815185387386009, + -0.04843989014625549, + 0.01417221873998642, + 0.026692505925893784, + 0.0023922978434711695, + -0.04369872435927391, + -0.08926654607057571, + 0.0026903150137513876, + -0.03696935251355171, + -0.002551637589931488, + 0.009431554935872555, + 0.026116130873560905, + -0.006958728190511465, + -0.002141412580385804, + 0.04574574530124664, + -0.0265728160738945, + 0.026631074026226997, + 0.038256410509347916, + -0.05301491916179657, + 0.003882748307660222, + 0.019330747425556183, + 0.022918039932847023, + -0.01216316968202591, + 0.06673868000507355, + -0.0437840037047863, + 0.07769957929849625, + -0.018716737627983093, + -0.05757647007703781, + -0.024929247796535492, + -0.0026189982891082764, + -0.07713627815246582, + 0.06504566967487335, + 0.03401261195540428, + -0.006688381545245647, + 0.00740303099155426, + -0.04742772504687309, + -0.0363510437309742, + 0.0021575281862169504, + 0.055273015052080154, + -0.01292359083890915, + 0.008398679085075855, + -0.017235547304153442, + -0.06933771818876266, + -0.023445632308721542, + 0.08455928415060043, + 0.02391771227121353, + 0.027352679520845413, + -0.06426014751195908, + -0.020447630435228348, + 0.005165628157556057, + 0.0178054291754961, + -0.009573280811309814, + -0.031630001962184906, + -0.01707049459218979, + -0.01566043682396412, + -0.016796711832284927, + 0.009098866954445839, + 0.032836299389600754, + -0.0733344629406929, + 0.06710889935493469, + 0.046053025871515274, + 0.02705153077840805, + -0.025517720729112625, + -0.012842105701565742, + -0.018786635249853134, + 0.004295899532735348, + -0.014369485899806023, + -0.06724471598863602, + -0.012760956771671772, + -0.01836458034813404, + -0.00929518137127161, + -0.05256617069244385, + -0.06008375063538551, + -0.0016317886766046286, + -0.045775312930345535, + 0.026287002488970757, + -0.04919085651636124, + -0.041538409888744354, + -0.03529488295316696, + 0.014668175019323826, + -0.034288439899683, + 0.03295760601758957, + -0.05478597432374954, + -0.004475493915379047, + 0.04592480510473251, + 0.06967853009700775, + 0.009266264736652374, + 0.001790371723473072, + 0.020226793363690376, + -0.05141701549291611, + -0.0011405664263293147, + 0.04043075814843178, + -0.08125229179859161, + -0.0032082980033010244, + -0.011070605367422104, + 0.006179114803671837, + 0.012523090466856956, + -0.015554573386907578, + -0.05098886415362358, + -0.050969772040843964, + -0.0764651894569397, + 0.010751461610198021, + -0.02323990687727928, + 0.03378252312541008, + 0.00987475086003542, + -0.0002837576321326196, + -0.02040388248860836, + -0.08074668794870377, + -0.0011368902632966638, + 0.007454159203916788, + -0.012466071173548698, + 0.03209332376718521, + 0.023357314988970757, + -0.04250813275575638, + -0.05197746679186821, + -0.0032767520751804113, + -0.016314717009663582, + 0.048089005053043365, + 0.01260228082537651, + -0.058030273765325546, + -0.020384544506669044, + -0.023983946070075035, + -0.010686483234167099, + -0.005069795995950699, + -0.04359026625752449, + -0.05281777307391167, + -0.0564735047519207, + 0.0211441982537508, + 0.03715534880757332, + 0.060332078486680984, + 0.04152520373463631, + -0.054246895015239716, + -0.030156394466757774, + -0.003449029754847288, + 0.004480309784412384, + -0.01001207996159792, + -0.014002281241118908, + 0.05839996412396431, + -0.025988027453422546, + 0.00859775673598051, + -0.006169992499053478, + -0.04511529207229614, + 0.005289457738399506, + -0.0036524732131510973, + -0.041411757469177246, + -0.049346283078193665, + -0.005228586029261351, + -0.08865255117416382, + -0.02881421335041523, + -0.04554671421647072, + 0.02887783758342266, + -0.043043773621320724, + 0.017193207517266273, + -0.007676500361412764, + -0.05310560017824173, + -0.014476086013019085, + 0.013972396962344646, + -0.007126152515411377, + -0.011438692919909954, + -0.0066301035694777966, + 0.035397548228502274, + -0.02895398996770382, + 0.0169553030282259, + -0.01257238071411848, + 0.038260091096162796, + 0.032011691480875015, + 0.0052822306752204895, + 0.13175901770591736, + 0.050640031695365906, + -0.0054465970024466515, + 0.010695336386561394, + 0.06630586087703705, + 0.010517016053199768, + -0.005450728815048933, + -0.06454240530729294, + 0.000673260074108839, + -0.03224816173315048 + ], + "paraphrase_query": [ + 0.07563937455415726, + 0.044951751828193665, + -0.10381697118282318, + 0.012933552265167236, + 0.013005409389734268, + 0.058970674872398376, + 0.010258755646646023, + 0.033107295632362366, + 0.06961818039417267, + 0.09382722526788712, + -0.017796039581298828, + 0.022032005712389946, + 0.040195684880018234, + 0.034211400896310806, + 0.01601613126695156, + 0.024841055274009705, + 0.01319377776235342, + -0.11746446788311005, + 0.0009046446066349745, + 0.025666886940598488, + -0.012587622739374638, + -0.018955081701278687, + -0.025525107979774475, + 0.08544657379388809, + 0.07781264930963516, + -0.053411856293678284, + 0.06148132309317589, + -0.0020504111889749765, + -0.005600812844932079, + -0.01944934017956257, + 0.02718743309378624, + 0.02670271322131157, + 0.019360274076461792, + -0.01822515018284321, + 0.005070455372333527, + -0.049727216362953186, + 0.00492201978340745, + 0.01196087896823883, + -0.0173896886408329, + -0.008157438598573208, + 0.06318145245313644, + 0.007090756203979254, + 0.026097839698195457, + -0.060507602989673615, + 0.023979119956493378, + 0.040770579129457474, + 0.06691893190145493, + -0.03056751936674118, + 0.04709409177303314, + -0.033934514969587326, + 0.03969186544418335, + -0.036404624581336975, + -0.02208634838461876, + -0.007897076196968555, + 0.05058363825082779, + 0.03667197376489639, + -0.006833284627646208, + -0.02428429387509823, + 0.03697933629155159, + -0.05644383281469345, + 0.07862889766693115, + 0.0536290742456913, + -0.03467759117484093, + 0.09294218569993973, + -0.013681392185389996, + -0.007312700618058443, + -0.02417200244963169, + 0.047237735241651535, + 0.019051970914006233, + -0.031070295721292496, + 0.04029080271720886, + 0.04899364709854126, + -0.023137057200074196, + -8.328058902407065e-05, + 0.025118917226791382, + 0.011438011191785336, + -0.031677886843681335, + -0.06930066645145416, + 0.01791590265929699, + 0.02572157047688961, + 0.03928546607494354, + -0.03751632943749428, + 0.08533425629138947, + -0.006163964048027992, + 0.056495748460292816, + 0.006545217242091894, + 0.010615103878080845, + -0.0664040818810463, + 0.010783147998154163, + 0.04262947663664818, + 0.028821879997849464, + 0.04445574805140495, + 0.02530664950609207, + 0.03570741042494774, + -0.004696797113865614, + 0.027375802397727966, + -0.03414176404476166, + 0.04063849523663521, + -0.03478386253118515, + 0.012062122114002705, + -0.04796784743666649, + -0.05177422985434532, + 0.0007525645778514445, + 0.020108819007873535, + -0.007159634958952665, + 0.010197145864367485, + -0.024377668276429176, + 0.007659323513507843, + 0.01993783377110958, + 0.031162826344370842, + -0.047724220901727676, + 0.016186915338039398, + -0.05764784663915634, + 0.006158775184303522, + 0.028938304632902145, + -0.026906725019216537, + 0.052644096314907074, + -0.01251925714313984, + -0.007624413352459669, + 0.025361113250255585, + -0.017252592369914055, + 0.009347162209451199, + 0.04851666837930679, + 0.06161468103528023, + -0.01354466937482357, + 0.042243842035532, + -0.046521175652742386, + 0.01772594265639782, + 0.05485351383686066, + -0.00907864235341549, + 0.01125884149223566, + 0.014469565823674202, + 0.006989903282374144, + 0.057976000010967255, + 0.01072631124407053, + 0.02697281539440155, + 0.0035090239252895117, + -0.02046002261340618, + -0.01583828218281269, + 0.005161571316421032, + 0.010418565012514591, + 0.044566985219717026, + -0.006639557890594006, + -0.005232523661106825, + -0.013978670351207256, + -0.04083247482776642, + 0.0332721546292305, + -0.02114553563296795, + -0.021824931725859642, + 0.02108292654156685, + 0.034964531660079956, + -0.0026458639185875654, + 0.04689545929431915, + -0.018293539062142372, + 0.018128810450434685, + -0.060194231569767, + -0.009149463847279549, + -0.025645749643445015, + 0.01570483110845089, + 0.035021696239709854, + 0.11010723561048508, + 0.043414995074272156, + -0.023554902523756027, + 0.09534033387899399, + -0.013336551375687122, + -0.031173264607787132, + 0.01998380944132805, + 0.027920475229620934, + -0.006440320052206516, + -0.002957393415272236, + 0.004268661141395569, + 0.0005243946798145771, + -0.01871560700237751, + -0.037807878106832504, + 0.06109987199306488, + 0.0011125373421236873, + 0.03851191699504852, + -0.04589145630598068, + 0.01602238416671753, + 0.0098087964579463, + 0.018891921266913414, + -0.031719233840703964, + 0.003025698708370328, + 0.08052512258291245, + 0.011434141546487808, + -0.007823314517736435, + 0.022474046796560287, + 0.05157781019806862, + 0.007183743640780449, + -0.02471798099577427, + -0.014461008831858635, + 0.0013540450017899275, + -0.06418993324041367, + 0.0018196816090494394, + -0.02881075069308281, + -0.026981834322214127, + 0.028839854523539543, + 0.04653911292552948, + 0.04917696863412857, + 0.015446021221578121, + -0.07377883791923523, + -0.0052611446008086205, + -0.05267489701509476, + 0.03250335156917572, + -0.03426516428589821, + 0.04788650944828987, + 0.01795877143740654, + -0.026003237813711166, + -0.022569801658391953, + -0.04821951314806938, + 0.07018709182739258, + -0.020979437977075577, + -0.020943589508533478, + -0.038944534957408905, + -0.005632980260998011, + -0.032181303948163986, + -0.02282395213842392, + -0.02384299412369728, + -0.04029428958892822, + 0.0036929247435182333, + 0.0031659158412367105, + -0.018661368638277054, + -0.021651580929756165, + -0.003017338225618005, + -0.006499530281871557, + -0.0031152551528066397, + -0.07412028312683105, + -0.042115505784749985, + -0.04514085873961449, + 0.029732022434473038, + 0.005966706667095423, + -0.09263350814580917, + 0.08039362728595734, + 0.016939999535679817, + -0.021387344226241112, + 0.03768729418516159, + -0.013050158508121967, + 0.0641629546880722, + -0.020355453714728355, + -0.019735073670744896, + -0.044457416981458664, + 0.05117696896195412, + 0.008409515023231506, + 0.014098098501563072, + -0.03815250098705292, + 0.003450468648225069, + 0.018487125635147095, + -0.03563212975859642, + -0.003873276524245739, + 0.012744561769068241, + -0.006239001173526049, + 0.009261157363653183, + 0.03228927403688431, + 0.029941922053694725, + 0.014849820174276829, + 0.040934838354587555, + -0.013557194732129574, + 0.009779052808880806, + -0.0015322305262088776, + -0.015656230971217155, + 0.02748945914208889, + -0.034379906952381134, + -0.016262460500001907, + -0.028691820800304413, + -0.0134021807461977, + -0.028270654380321503, + -0.05666399374604225, + -0.04146397486329079, + 0.044550079852342606, + 0.006174638867378235, + 0.022784817963838577, + 0.06264983862638474, + -0.04478584602475166, + 0.030265530571341515, + -0.015299376100301743, + -0.022669805213809013, + -0.0020824193488806486, + -0.034294065088033676, + -0.02943631261587143, + 0.06973370164632797, + -0.005823781248182058, + 0.00875718705356121, + -0.036070991307497025, + -0.02585827186703682, + 0.02695818617939949, + 0.036274004727602005, + 0.03920917958021164, + -0.009136143140494823, + -0.03199266642332077, + 0.023350980132818222, + -0.020649420097470284, + 0.050007157027721405, + -0.016069568693637848, + 0.07433676719665527, + 0.024353794753551483, + 0.08551505953073502, + 0.0012566486839205027, + 0.021516965702176094, + -0.014606218785047531, + -0.024875149130821228, + 0.05238021910190582, + -0.0027186411898583174, + 0.034511905163526535, + 0.04690590128302574, + 0.0016458206810057163, + -0.045229505747556686, + 0.029715977609157562, + 0.006758865900337696, + 0.05240838974714279, + -0.03524618595838547, + -0.06741555780172348, + -0.05185435339808464, + -0.027453437447547913, + 0.04658479616045952, + -0.04837347939610481, + 0.05925726518034935, + 0.0426187664270401, + -0.009259475395083427, + 0.03507320210337639, + -0.005243795458227396, + 0.007778884842991829, + 0.026350660249590874, + -0.005177740007638931, + 0.011640103533864021, + 0.027170615270733833, + -0.02501584030687809, + 0.009011970832943916, + 0.03697763383388519, + -0.022232329472899437, + -0.027736404910683632, + -0.003668621415272355, + 0.04178178682923317, + 0.059150490909814835, + -0.03573083505034447, + -0.00939685758203268, + 0.004975534975528717, + 0.01002561580389738, + 0.005373264197260141, + -0.031511340290308, + 0.052563101053237915, + 0.09675197303295135, + -0.07258515059947968, + 0.014194052666425705, + -0.05187372490763664, + -0.006363960914313793, + 0.009313041344285011, + -0.0020671971142292023, + -0.004487851168960333, + 0.055991217494010925, + 0.0003737418446689844, + -0.016131456941366196, + 0.015771040692925453, + -0.007459891028702259, + -0.035391926765441895, + -0.0013215638464316726, + -0.0004980667727068067, + 0.029820475727319717, + 0.06558115780353546, + -0.007725974079221487, + -0.046034928411245346, + 0.0453345812857151, + -0.03393309935927391, + 0.026703698560595512, + -0.004847528412938118, + 0.020458459854125977, + 0.04740547761321068, + 0.009288874454796314, + 0.023204203695058823, + 0.002336920937523246, + 0.040167830884456635, + -0.009001133032143116, + -0.055470872670412064, + 0.024002179503440857, + 0.05031982809305191, + -0.023971762508153915, + 0.01419965922832489, + -0.06634949892759323, + -0.015103098936378956, + 0.01842097006738186, + 0.015711180865764618, + 0.04222189262509346, + -0.0033563815522938967, + -0.021083075553178787, + -0.03687334060668945, + -0.04226105287671089, + -0.03761494159698486, + 0.046930886805057526, + 0.011943898163735867, + -0.0159631185233593, + -0.01229014340788126, + 0.018807798624038696, + -0.040775854140520096, + -0.05883694812655449, + -0.039298444986343384, + -0.004135280381888151, + -0.0035059810616075993, + -0.0046470677480101585, + -0.05639946088194847, + 0.03698357939720154, + -0.0006696773925796151, + -0.051650434732437134, + 0.03416513651609421, + -0.004219100810587406, + 0.05078054964542389, + -0.029774151742458344, + 0.05430423468351364, + -0.02917175181210041, + -0.05793788656592369, + 0.046401966363191605, + 0.02044791355729103, + -0.015234327875077724, + 0.0816413164138794, + 0.014461847022175789, + 0.016794376075267792, + -0.034356385469436646, + 0.043045349419116974, + -0.012729162350296974, + -0.04840034618973732, + 0.022931141778826714, + -0.009672458283603191, + 0.03710097074508667, + 0.002497660228982568, + 0.021295487880706787, + -0.01081894151866436, + -0.005114822182804346, + 0.06666533648967743, + 0.017743553966283798, + -0.028922230005264282, + -0.026583783328533173, + -0.06231055036187172, + 0.014454836025834084, + 0.032214853912591934, + 0.03254424035549164, + -0.01980658993124962, + -0.06288133561611176, + 0.027997247874736786, + 0.008878464810550213, + 0.01267779991030693, + 0.0014761538477614522, + 0.03722553700208664, + 0.011402842588722706, + -0.04471265897154808, + -0.03485701233148575, + -0.021652698516845703, + 0.015706807374954224, + 0.08832989633083344, + 0.02573048695921898, + -0.03340991213917732, + -0.011327791027724743, + 0.020303020253777504, + 0.022442644461989403, + 0.03588799014687538, + 0.04263828322291374, + -0.007532990537583828, + 0.052800387144088745, + -0.02855009213089943, + -0.02815694734454155, + 0.025533264502882957, + 0.003541866084560752, + 0.04024374485015869, + 0.0884997770190239, + -7.200983236543834e-05, + 0.00933838076889515, + 0.008277008309960365, + 0.011751553043723106, + 0.000723029428627342, + -0.017995361238718033, + 0.002157199662178755, + 0.03566899523139, + 0.0482158288359642, + -0.06126498430967331, + 0.004130835644900799, + -0.0018153556156903505, + 0.0009250106522813439, + -0.04025152698159218, + 0.0048891836777329445, + -0.018460016697645187, + -0.043958667665719986, + -0.028415793552994728, + 0.04847021773457527, + -0.03714435175061226, + 0.05003475770354271, + 0.008552907966077328, + -0.046494223177433014, + 0.03503710776567459, + 0.03965332731604576, + -0.01090865395963192, + 0.0006830740603618324, + 0.023895621299743652, + 0.032841358333826065, + -0.006451840046793222, + 0.0015266984701156616, + -0.01150133553892374, + -0.0344635546207428, + -0.012013253755867481, + -0.05031405761837959, + -0.04383353888988495, + 0.0440698079764843, + 0.02791421487927437, + 0.031319454312324524, + -0.02147885411977768, + 0.06832845509052277, + -0.022052478045225143, + 0.012407501228153706, + -0.00496590556576848, + -0.05594398081302643, + -0.013196195475757122, + -0.043665122240781784, + -0.03327406942844391, + 0.009732309728860855, + -0.02305634133517742, + -0.00556295458227396, + -0.011641213670372963, + -0.008279677480459213, + 0.0281533095985651, + -0.016804533079266548, + 0.005843710154294968, + -0.006825277581810951, + -0.10277707129716873, + 0.021407680585980415, + 0.028206514194607735, + -0.00902584008872509, + -0.013780699111521244, + 0.005261505022644997, + -0.08816658705472946, + -0.03573956713080406, + 0.0008514812798239291, + -0.06588489562273026, + 0.03181877359747887, + 0.005183267407119274, + 0.04062454402446747, + 0.061462100595235825, + -0.06779246032238007, + -0.018221789970993996, + -0.010236603207886219, + 0.00047183316200971603, + -0.06446095556020737, + -0.021007254719734192, + -0.004122115671634674, + 0.007595954928547144, + -0.013504821807146072, + 0.03754039481282234, + -0.0195633415132761, + 0.012601318769156933, + -0.005179057829082012, + -0.009105146862566471, + -0.018324123695492744, + -0.01846819370985031, + -0.043171003460884094, + -0.03436756879091263, + 0.004700302612036467, + -0.11366402357816696, + -0.03343813493847847, + -0.030404208227992058, + 0.016984110698103905, + -0.09927711635828018, + 0.02544722706079483, + -0.004188238177448511, + -0.037437181919813156, + -0.014028574340045452, + 0.028661496937274933, + -0.019399438053369522, + 0.04556034505367279, + -0.011241859756410122, + -0.007852704264223576, + -0.005927511025220156, + 0.012734543532133102, + -0.04608645662665367, + -0.0270491112023592, + 0.049331940710544586, + 0.03236677870154381, + -0.013122049160301685, + -0.025352830067276955, + -0.014006780460476875, + -0.01746532693505287, + 0.006426376290619373, + -0.016809364780783653, + -0.06374692171812057, + -0.039252378046512604, + -0.03619817644357681, + -0.04859007149934769, + -0.0008585317991673946, + 0.019318504258990288, + -0.0011830421863123775, + 0.01588822342455387, + -0.014229823835194111, + -0.028481462970376015, + -0.03824499249458313, + 0.010830463841557503, + 0.011207794770598412, + -0.01626027561724186, + -0.009966238401830196, + -0.027257444337010384, + -0.007319690193980932, + 0.019873710349202156, + 0.003637217218056321, + -0.050996582955121994, + -0.10067758709192276, + -0.0006862303125672042, + -0.01225066464394331, + -0.044326648116111755, + 0.028829412534832954, + 0.003080033464357257, + -0.02066786028444767, + 0.044365882873535156, + 0.05013382434844971, + 0.01509327907115221, + 0.036207493394613266, + 0.03466777503490448, + -0.05513041466474533, + 0.0044077616184949875, + 0.04204444959759712, + -0.028776604682207108, + -0.03674590587615967, + 0.06341822445392609, + -0.013921674340963364, + 0.03736335411667824, + -0.04223078861832619, + -0.03534340485930443, + 0.008640135638415813, + 0.04155464842915535, + -0.03555804118514061, + 0.08126675337553024, + 0.02833716943860054, + 0.04088519141077995, + -0.009072018787264824, + -0.04347702115774155, + -0.04663033410906792, + -0.0335489921271801, + 0.05593689903616905, + -0.012346108444035053, + 0.02474166825413704, + -0.03207416087388992, + -0.0404789075255394, + -0.028464827686548233, + 0.03685234859585762, + 0.012776107527315617, + 0.010501091368496418, + -0.03657430410385132, + -0.009618331678211689, + 0.02421198971569538, + 0.017296770587563515, + -0.03731854259967804, + 0.020054319873452187, + 0.019713588058948517, + 0.05606694892048836, + -0.010878914967179298, + 0.009339292533695698, + 0.016856281086802483, + -0.023483699187636375, + 0.08042444288730621, + 0.03234444558620453, + 0.00710707064718008, + -0.025376247242093086, + -0.016342181712388992, + 0.050446998327970505, + 0.018679143860936165, + -0.05224505439400673, + -0.03415687382221222, + -0.03950631991028786, + 0.002496677916496992, + -0.02331732213497162, + 0.001193678006529808, + -0.030446568503975868, + -0.008125221356749535, + -0.03103300929069519, + 0.0006478068535216153, + -0.025568373501300812, + -0.05977100878953934, + -0.02729092352092266, + 0.007304736878722906, + -0.03388095647096634, + -0.008483147248625755, + -0.06122725456953049, + -0.0022162890527397394, + 0.012101683765649796, + 0.04364695027470589, + -0.0254487544298172, + 0.020212428644299507, + -0.005720080807805061, + -0.03848709166049957, + -0.014168472960591316, + 0.046762943267822266, + -0.04193192720413208, + -0.006753803230822086, + -0.04597880691289902, + -0.01633371412754059, + 0.003253790084272623, + -0.03459026664495468, + -0.021968230605125427, + -0.08774406462907791, + -0.10138633102178574, + -0.02059061825275421, + -0.03406858816742897, + 0.026204269379377365, + -0.0033026800956577063, + 0.02136273682117462, + -0.005489582195878029, + -0.027565238997340202, + 0.04576081037521362, + 0.031919680535793304, + -0.0008669350063428283, + 0.003675919258967042, + 0.04017110541462898, + -0.0407070554792881, + -0.025028077885508537, + 0.00940376240760088, + 0.0012068490032106638, + 0.0281363595277071, + -0.0022061483468860388, + 0.02158242091536522, + 0.008484560996294022, + -0.04596230387687683, + 0.03815562650561333, + 0.006286890711635351, + -0.026988815516233444, + -0.06016876548528671, + -0.02577892318367958, + 0.0015845522284507751, + -0.01251696515828371, + 0.08260448276996613, + 0.0033863605931401253, + -0.11704328656196594, + -0.06068435683846474, + -0.02884758450090885, + -0.011466464027762413, + -0.004658904857933521, + -0.010832620784640312, + 0.03050537221133709, + -0.027887847274541855, + 0.0010878940811380744, + -0.025875523686408997, + -0.053577911108732224, + 0.01279984600841999, + -0.026293445378541946, + -0.015506679192185402, + -0.07301454991102219, + -0.00017940637189894915, + -0.05557995289564133, + -0.03323184326291084, + -0.031889986246824265, + 0.010983175598084927, + -0.02527605928480625, + -0.013710030354559422, + 0.020160764455795288, + -0.04108891263604164, + 0.006233372259885073, + 0.03370879963040352, + -0.0004395705764181912, + -0.021134912967681885, + -0.005690909456461668, + -0.004662687424570322, + -0.0074738627299666405, + 0.022959647700190544, + 0.0008352036820724607, + 0.04038509353995323, + 0.012161842547357082, + -0.029094338417053223, + 0.09499742835760117, + 0.042413949966430664, + 0.012602478265762329, + -0.0034536977764219046, + 0.06749358028173447, + 0.034492626786231995, + -0.04123144969344139, + -0.03784525766968727, + -0.01001029647886753, + 0.04269317165017128 + ] + } +} diff --git a/python/tests/unit/memory/recorded_nomic_relevance_embeddings.py b/python/tests/unit/memory/recorded_nomic_relevance_embeddings.py new file mode 100644 index 000000000000..e86e236ae942 --- /dev/null +++ b/python/tests/unit/memory/recorded_nomic_relevance_embeddings.py @@ -0,0 +1,8 @@ +# Copyright (c) Microsoft. All rights reserved. +"""Offline vectors generated by generate_recorded_nomic_relevance_embeddings.py.""" + +import json +from pathlib import Path + +FIXTURE = json.loads(Path(__file__).with_suffix(".json").read_text()) +RECORDED_EMBEDDINGS = {text: FIXTURE["vectors"][name] for name, text in FIXTURE["texts"].items()} diff --git a/python/tests/unit/memory/test_text_memory_plugin_relevance.py b/python/tests/unit/memory/test_text_memory_plugin_relevance.py new file mode 100644 index 000000000000..6fb4555e66fd --- /dev/null +++ b/python/tests/unit/memory/test_text_memory_plugin_relevance.py @@ -0,0 +1,53 @@ +# Copyright (c) Microsoft. All rights reserved. + +import numpy as np + +from semantic_kernel.connectors.ai.embedding_generator_base import EmbeddingGeneratorBase +from semantic_kernel.core_plugins.text_memory_plugin import TextMemoryPlugin +from semantic_kernel.memory.semantic_text_memory import SemanticTextMemory +from semantic_kernel.memory.volatile_memory_store import VolatileMemoryStore +from tests.unit.memory.recorded_nomic_relevance_embeddings import RECORDED_EMBEDDINGS + +STORED_TEXT = "Before deleting user data, require explicit confirmation." +NEGATED_QUERY = "Before deleting user data, do not require explicit confirmation." +PARAPHRASE_STORED_TEXT = "Erase records only after the subject has authorized the action." +PARAPHRASE_QUERY = "Obtain affirmative consent from the person before erasing any of their personal information." + + +class RecordedEmbeddingGenerator(EmbeddingGeneratorBase): + async def generate_embeddings(self, texts, settings=None, **kwargs): + return np.array([RECORDED_EMBEDDINGS[text] for text in texts], dtype=np.float32) + + +async def test_recall_default_threshold_returns_negated_instruction(): + memory = SemanticTextMemory(VolatileMemoryStore(), RecordedEmbeddingGenerator(ai_model_id="recorded-nomic")) + plugin = TextMemoryPlugin(memory) + + await plugin.save(STORED_TEXT, "stored") + + result = await plugin.recall(NEGATED_QUERY) + + assert result == STORED_TEXT + + +async def test_recall_default_threshold_filters_equivalent_paraphrase(): + memory = SemanticTextMemory(VolatileMemoryStore(), RecordedEmbeddingGenerator(ai_model_id="recorded-nomic")) + plugin = TextMemoryPlugin(memory) + + await plugin.save(PARAPHRASE_STORED_TEXT, "stored") + + result = await plugin.recall(PARAPHRASE_QUERY) + + assert result == "" + + +def test_recorded_vectors_cross_the_default_relevance_threshold(): + stored = np.array(RECORDED_EMBEDDINGS[STORED_TEXT]) + negated = np.array(RECORDED_EMBEDDINGS[NEGATED_QUERY]) + paraphrase_stored = np.array(RECORDED_EMBEDDINGS[PARAPHRASE_STORED_TEXT]) + paraphrase_query = np.array(RECORDED_EMBEDDINGS[PARAPHRASE_QUERY]) + + assert np.dot(stored, negated) / (np.linalg.norm(stored) * np.linalg.norm(negated)) > 0.75 + assert np.dot(paraphrase_stored, paraphrase_query) / ( + np.linalg.norm(paraphrase_stored) * np.linalg.norm(paraphrase_query) + ) < 0.75