diff --git a/research/t16-audit-mutant-coverage/EXPECTED.json b/research/t16-audit-mutant-coverage/EXPECTED.json new file mode 100644 index 0000000000..d69c82a030 --- /dev/null +++ b/research/t16-audit-mutant-coverage/EXPECTED.json @@ -0,0 +1,1263 @@ +{ + "accepted_fields": { + "async_schedule": [], + "async_schedule.scope": [], + "async_schedule.source_plan_sha256": [], + "cover": [], + "expanded_scalar_interface": [], + "hosting": [], + "hosting.idle_remainder_semantics": [], + "hosting.owner_map_sha256": [], + "hosting.pooled_slots.0.time": [], + "hosting.rule": [], + "hosting.scope": [], + "loop_order": [], + "m": [], + "port_restriction": [], + "provider_dispatch": [], + "replica_lift": [], + "route_semantics": [], + "supplier_substitution": [], + "version": [] + }, + "accepted_new_mutants": [ + "pooled slot time changed" + ], + "assert_lines_in_audit": 42, + "assert_lines_reached_by_any": [ + 93, + 94, + 95, + 96, + 97, + 100, + 101, + 103, + 108, + 111, + 112, + 114, + 119, + 120, + 121, + 122, + 129, + 130, + 131, + 132, + 148, + 149, + 151, + 152, + 153, + 156, + 159, + 163, + 169, + 171, + 172, + 173, + 178, + 181 + ], + "assert_lines_reached_by_shipped": [ + 93, + 94, + 111, + 149, + 151, + 152, + 159, + 181 + ], + "assert_lines_unreached": [ + 98, + 123, + 157, + 160, + 161, + 164, + 167, + 182 + ], + "audit_script_sha256": "1467a2acb654e62552c69fb9a7c4c85f70a02120881d222313865058ab71a6cc", + "coverage": { + "100": { + "field": 1, + "new": 1, + "shipped": 0, + "statement": "src=parent['execution_schedule'];assert t['async_schedule']['source_schedule']==src" + }, + "101": { + "field": 1, + "new": 0, + "shipped": 0, + "statement": "assert t['async_schedule']['source_call_order']==[[c['stage'],c['replica']]for c in parent['calls']]" + }, + "103": { + "field": 3, + "new": 1, + "shipped": 0, + "statement": "assert [list(x)for x in order]==t['async_schedule']['order'] and len(set(order))==144" + }, + "108": { + "field": 1, + "new": 0, + "shipped": 0, + "statement": "r=e['replica'];assert 0<=r<16 and j<30;perrep[r].append(j)" + }, + "111": { + "field": 32, + "new": 2, + "shipped": 1, + "statement": "assert e==expected" + }, + "112": { + "field": 5, + "new": 0, + "shipped": 0, + "statement": "assert 0<=e['time']<144 and order[e['time']][1]==r" + }, + "114": { + "field": 0, + "new": 1, + "shipped": 0, + "statement": "assert order[e['time']]==(e['stage'],r);all_local.append((e['stage'],r,e['time']))" + }, + "119": { + "field": 9, + "new": 1, + "shipped": 0, + "statement": "assert e==expected" + }, + "120": { + "field": 0, + "new": 3, + "shipped": 0, + "statement": "assert all(v==list(range(30))for v in perrep.values()) and globals_==[30,31]" + }, + "121": { + "field": 0, + "new": 1, + "shipped": 0, + "statement": "assert all_local==[(st,r,i)for i,(st,r)in enumerate(order)]" + }, + "122": { + "field": 0, + "new": 1, + "shipped": 0, + "statement": "assert all('replica'in e for e in t['execution_schedule'][:-2])" + }, + "123": { + "field": 0, + "new": 0, + "shipped": 0, + "statement": "assert [e['source_event']for e in t['execution_schedule'][-2:]]==[30,31]" + }, + "129": { + "field": 0, + "new": 1, + "shipped": 0, + "statement": "assert e['time']==block" + }, + "130": { + "field": 1, + "new": 1, + "shipped": 0, + "statement": "assert t['bank_padding']==dict(parent['bank_padding'],families=[],children=[])" + }, + "131": { + "field": 1, + "new": 0, + "shipped": 0, + "statement": "assert t['cover_final_correction']==dict(parent['cover_final_correction'],families=[0,DATA],charged_calls_per_cover=DATA)" + }, + "132": { + "field": 1, + "new": 1, + "shipped": 0, + "statement": "assert t['private_namespace']=={'centers':[DATA,DATA+CENTRES],'reserved_tableau':[DATA+CENTRES,DATA+CENTRES+R],'coefficient':[DATA+CENTRES+R,PHYSICAL]}" + }, + "148": { + "field": 3, + "new": 0, + "shipped": 0, + "statement": "assert {k:v for k,v in call.items()if k not in('owners','time','provider_id','local_program_sha256')}=={k:v for k,v in old.items()if k not in('owners','local_pr" + }, + "149": { + "field": 1, + "new": 0, + "shipped": 1, + "statement": "pid=expected_provider(call['stage'],call['replica']);assert call['provider_id']==pid" + }, + "151": { + "field": 1, + "new": 0, + "shipped": 1, + "statement": "assert call['local_program_sha256']==checks[call['orientation']]['sha256']" + }, + "152": { + "field": 2, + "new": 1, + "shipped": 3, + "statement": "s,rep,pair=call['stage'],call['replica'],call['pair'];assert call['time']==t['async_schedule']['order'].index([s,rep]);own=call['owners'];assert len(own)==N and" + }, + "153": { + "field": 0, + "new": 1, + "shipped": 0, + "statement": "assert own[:(2*V)]==old['owners'][:(2*V)]" + }, + "156": { + "field": 0, + "new": 4, + "shipped": 0, + "statement": "assert windows[ownerrep,i][0]2*max(paid)**2" + }, + "93": { + "field": 2, + "new": 0, + "shipped": 1, + "statement": "assert(t['live'],t['physical'])==(DATA,PHYSICAL)" + }, + "94": { + "field": 2, + "new": 2, + "shipped": 4, + "statement": "assert t['local_providers']==registry and t['recipe']==parent['recipe']and 'each local invocation completes before the next' in t['loop_order'].lower()" + }, + "95": { + "field": 0, + "new": 1, + "shipped": 0, + "statement": "assert not any(k in t for k in('local_programs','exact_local_clifford_recipe_sha256','residual_adapter_sha256'))" + }, + "96": { + "field": 3, + "new": 2, + "shipped": 0, + "statement": "assert len(t['calls'])==144 and t['hosting']['max_helpers_per_physical_family_per_call']==1 and t['hosting']['banks']==[0]*9" + }, + "97": { + "field": 2, + "new": 0, + "shipped": 0, + "statement": "for key in('exact_data_phase_recipe_sha256','adapter_policy'):assert t[key]==parent[key]" + }, + "98": { + "field": 0, + "new": 0, + "shipped": 0, + "statement": "assert t['exact_data_phase_recipe_sha256']==sha(BASE/'work/complex-nine-native/DATA-PHASES.json')" + } + }, + "field_census": { + "adapter_policy": { + "exception": "AssertionError", + "line": 97, + "result": "REJECTED", + "statement": "for key in('exact_data_phase_recipe_sha256','adapter_policy'):assert t[key]==parent[key]" + }, + "async_schedule": { + "result": "ACCEPTED" + }, + "async_schedule.order": { + "exception": "AssertionError", + "line": 103, + "result": "REJECTED", + "statement": "assert [list(x)for x in order]==t['async_schedule']['order'] and len(set(order))==144" + }, + "async_schedule.scope": { + "result": "ACCEPTED" + }, + "async_schedule.source_call_order": { + "exception": "AssertionError", + "line": 101, + "result": "REJECTED", + "statement": "assert t['async_schedule']['source_call_order']==[[c['stage'],c['replica']]for c in parent['calls']]" + }, + "async_schedule.source_plan_sha256": { + "result": "ACCEPTED" + }, + "async_schedule.source_schedule": { + "exception": "AssertionError", + "line": 100, + "result": "REJECTED", + "statement": "src=parent['execution_schedule'];assert t['async_schedule']['source_schedule']==src" + }, + "bank_padding": { + "exception": "AssertionError", + "line": 130, + "result": "REJECTED", + "statement": "assert t['bank_padding']==dict(parent['bank_padding'],families=[],children=[])" + }, + "calls": { + "exception": "AssertionError", + "line": 96, + "result": "REJECTED", + "statement": "assert len(t['calls'])==144 and t['hosting']['max_helpers_per_physical_family_per_call']==1 and t['hosting']['banks']==[0]*9" + }, + "calls.0.local_program_sha256": { + "exception": "AssertionError", + "line": 151, + "result": "REJECTED", + "statement": "assert call['local_program_sha256']==checks[call['orientation']]['sha256']" + }, + "calls.0.orientation": { + "exception": "AssertionError", + "line": 148, + "result": "REJECTED", + "statement": "assert {k:v for k,v in call.items()if k not in('owners','time','provider_id','local_program_sha256')}=={k:v for k,v in old.items()if k not in('owners','local_pr" + }, + "calls.0.owners": { + "exception": "AssertionError", + "line": 152, + "result": "REJECTED", + "statement": "s,rep,pair=call['stage'],call['replica'],call['pair'];assert call['time']==t['async_schedule']['order'].index([s,rep]);own=call['owners'];assert len(own)==N and" + }, + "calls.0.pair": { + "exception": "AssertionError", + "line": 148, + "result": "REJECTED", + "statement": "assert {k:v for k,v in call.items()if k not in('owners','time','provider_id','local_program_sha256')}=={k:v for k,v in old.items()if k not in('owners','local_pr" + }, + "calls.0.provider_id": { + "exception": "AssertionError", + "line": 149, + "result": "REJECTED", + "statement": "pid=expected_provider(call['stage'],call['replica']);assert call['provider_id']==pid" + }, + "calls.0.replica": { + "exception": "AssertionError", + "line": 103, + "result": "REJECTED", + "statement": "assert [list(x)for x in order]==t['async_schedule']['order'] and len(set(order))==144" + }, + "calls.0.stage": { + "exception": "AssertionError", + "line": 103, + "result": "REJECTED", + "statement": "assert [list(x)for x in order]==t['async_schedule']['order'] and len(set(order))==144" + }, + "calls.0.time": { + "exception": "AssertionError", + "line": 152, + "result": "REJECTED", + "statement": "s,rep,pair=call['stage'],call['replica'],call['pair'];assert call['time']==t['async_schedule']['order'].index([s,rep]);own=call['owners'];assert len(own)==N and" + }, + "calls.0.window": { + "exception": "AssertionError", + "line": 148, + "result": "REJECTED", + "statement": "assert {k:v for k,v in call.items()if k not in('owners','time','provider_id','local_program_sha256')}=={k:v for k,v in old.items()if k not in('owners','local_pr" + }, + "cover": { + "result": "ACCEPTED" + }, + "cover_final_correction": { + "exception": "AssertionError", + "line": 131, + "result": "REJECTED", + "statement": "assert t['cover_final_correction']==dict(parent['cover_final_correction'],families=[0,DATA],charged_calls_per_cover=DATA)" + }, + "exact_data_phase_recipe_sha256": { + "exception": "AssertionError", + "line": 97, + "result": "REJECTED", + "statement": "for key in('exact_data_phase_recipe_sha256','adapter_policy'):assert t[key]==parent[key]" + }, + "execution_schedule": { + "exception": "TypeError", + "line": 106, + "result": "REJECTED", + "statement": "j=e['source_event'];old=copy.deepcopy(src[j])" + }, + "execution_schedule.0.entry_exit_phase_adapters [local_cover_sweep]": { + "exception": "AssertionError", + "line": 111, + "result": "REJECTED", + "statement": "assert e==expected" + }, + "execution_schedule.0.kind [local_cover_sweep]": { + "exception": "AssertionError", + "line": 111, + "result": "REJECTED", + "statement": "assert e==expected" + }, + "execution_schedule.0.provider_id [local_cover_sweep]": { + "exception": "AssertionError", + "line": 111, + "result": "REJECTED", + "statement": "assert e==expected" + }, + "execution_schedule.0.replica [local_cover_sweep]": { + "exception": "AssertionError", + "line": 111, + "result": "REJECTED", + "statement": "assert e==expected" + }, + "execution_schedule.0.replicas [local_cover_sweep]": { + "exception": "AssertionError", + "line": 111, + "result": "REJECTED", + "statement": "assert e==expected" + }, + "execution_schedule.0.source_event [local_cover_sweep]": { + "exception": "AssertionError", + "line": 111, + "result": "REJECTED", + "statement": "assert e==expected" + }, + "execution_schedule.0.stage [local_cover_sweep]": { + "exception": "AssertionError", + "line": 111, + "result": "REJECTED", + "statement": "assert e==expected" + }, + "execution_schedule.0.time [local_cover_sweep]": { + "exception": "AssertionError", + "line": 112, + "result": "REJECTED", + "statement": "assert 0<=e['time']<144 and order[e['time']][1]==r" + }, + "execution_schedule.232.kind [terminal_Y_translations]": { + "exception": "AssertionError", + "line": 111, + "result": "REJECTED", + "statement": "assert e==expected" + }, + "execution_schedule.232.pairs [terminal_Y_translations]": { + "exception": "AssertionError", + "line": 111, + "result": "REJECTED", + "statement": "assert e==expected" + }, + "execution_schedule.232.replica [terminal_Y_translations]": { + "exception": "AssertionError", + "line": 111, + "result": "REJECTED", + "statement": "assert e==expected" + }, + "execution_schedule.232.replicas [terminal_Y_translations]": { + "exception": "AssertionError", + "line": 111, + "result": "REJECTED", + "statement": "assert e==expected" + }, + "execution_schedule.232.source_event [terminal_Y_translations]": { + "exception": "AssertionError", + "line": 111, + "result": "REJECTED", + "statement": "assert e==expected" + }, + "execution_schedule.232.time [terminal_Y_translations]": { + "exception": "AssertionError", + "line": 112, + "result": "REJECTED", + "statement": "assert 0<=e['time']<144 and order[e['time']][1]==r" + }, + "execution_schedule.232.vector [terminal_Y_translations]": { + "exception": "AssertionError", + "line": 111, + "result": "REJECTED", + "statement": "assert e==expected" + }, + "execution_schedule.233.kind [terminal]": { + "exception": "AssertionError", + "line": 111, + "result": "REJECTED", + "statement": "assert e==expected" + }, + "execution_schedule.233.pairs [terminal]": { + "exception": "AssertionError", + "line": 111, + "result": "REJECTED", + "statement": "assert e==expected" + }, + "execution_schedule.233.replica [terminal]": { + "exception": "AssertionError", + "line": 111, + "result": "REJECTED", + "statement": "assert e==expected" + }, + "execution_schedule.233.replicas [terminal]": { + "exception": "AssertionError", + "line": 111, + "result": "REJECTED", + "statement": "assert e==expected" + }, + "execution_schedule.233.source_event [terminal]": { + "exception": "AssertionError", + "line": 108, + "result": "REJECTED", + "statement": "r=e['replica'];assert 0<=r<16 and j<30;perrep[r].append(j)" + }, + "execution_schedule.233.time [terminal]": { + "exception": "AssertionError", + "line": 112, + "result": "REJECTED", + "statement": "assert 0<=e['time']<144 and order[e['time']][1]==r" + }, + "execution_schedule.480.children_source [bank_padding_cover_sweep]": { + "exception": "AssertionError", + "line": 119, + "result": "REJECTED", + "statement": "assert e==expected" + }, + "execution_schedule.480.families [bank_padding_cover_sweep]": { + "exception": "AssertionError", + "line": 119, + "result": "REJECTED", + "statement": "assert e==expected" + }, + "execution_schedule.480.kind [bank_padding_cover_sweep]": { + "exception": "AssertionError", + "line": 119, + "result": "REJECTED", + "statement": "assert e==expected" + }, + "execution_schedule.480.source_event [bank_padding_cover_sweep]": { + "exception": "AssertionError", + "line": 119, + "result": "REJECTED", + "statement": "assert e==expected" + }, + "execution_schedule.481.charged_calls_per_cover [translate_every_live_family]": { + "exception": "AssertionError", + "line": 119, + "result": "REJECTED", + "statement": "assert e==expected" + }, + "execution_schedule.481.families [translate_every_live_family]": { + "exception": "AssertionError", + "line": 119, + "result": "REJECTED", + "statement": "assert e==expected" + }, + "execution_schedule.481.formula [translate_every_live_family]": { + "exception": "AssertionError", + "line": 119, + "result": "REJECTED", + "statement": "assert e==expected" + }, + "execution_schedule.481.kind [translate_every_live_family]": { + "exception": "AssertionError", + "line": 119, + "result": "REJECTED", + "statement": "assert e==expected" + }, + "execution_schedule.481.must_follow_bank_padding_and_data_signed_exchange [translate_every_live_family]": { + "exception": "AssertionError", + "line": 119, + "result": "REJECTED", + "statement": "assert e==expected" + }, + "execution_schedule.481.source_event [translate_every_live_family]": { + "exception": "IndexError", + "line": 106, + "result": "REJECTED", + "statement": "j=e['source_event'];old=copy.deepcopy(src[j])" + }, + "execution_schedule.5.child_indices [idle_cover_sweep]": { + "exception": "AssertionError", + "line": 111, + "result": "REJECTED", + "statement": "assert e==expected" + }, + "execution_schedule.5.from_level [idle_cover_sweep]": { + "exception": "AssertionError", + "line": 111, + "result": "REJECTED", + "statement": "assert e==expected" + }, + "execution_schedule.5.kind [idle_cover_sweep]": { + "exception": "AssertionError", + "line": 111, + "result": "REJECTED", + "statement": "assert e==expected" + }, + "execution_schedule.5.pair [idle_cover_sweep]": { + "exception": "AssertionError", + "line": 111, + "result": "REJECTED", + "statement": "assert e==expected" + }, + "execution_schedule.5.replica [idle_cover_sweep]": { + "exception": "AssertionError", + "line": 111, + "result": "REJECTED", + "statement": "assert e==expected" + }, + "execution_schedule.5.replicas [idle_cover_sweep]": { + "exception": "AssertionError", + "line": 111, + "result": "REJECTED", + "statement": "assert e==expected" + }, + "execution_schedule.5.source_event [idle_cover_sweep]": { + "exception": "AssertionError", + "line": 111, + "result": "REJECTED", + "statement": "assert e==expected" + }, + "execution_schedule.5.time [idle_cover_sweep]": { + "exception": "AssertionError", + "line": 112, + "result": "REJECTED", + "statement": "assert 0<=e['time']<144 and order[e['time']][1]==r" + }, + "execution_schedule.5.to_level [idle_cover_sweep]": { + "exception": "AssertionError", + "line": 111, + "result": "REJECTED", + "statement": "assert e==expected" + }, + "execution_schedule.6.coefficient [bridge]": { + "exception": "AssertionError", + "line": 111, + "result": "REJECTED", + "statement": "assert e==expected" + }, + "execution_schedule.6.dst [bridge]": { + "exception": "AssertionError", + "line": 111, + "result": "REJECTED", + "statement": "assert e==expected" + }, + "execution_schedule.6.kind [bridge]": { + "exception": "AssertionError", + "line": 111, + "result": "REJECTED", + "statement": "assert e==expected" + }, + "execution_schedule.6.replica [bridge]": { + "exception": "AssertionError", + "line": 111, + "result": "REJECTED", + "statement": "assert e==expected" + }, + "execution_schedule.6.replicas [bridge]": { + "exception": "AssertionError", + "line": 111, + "result": "REJECTED", + "statement": "assert e==expected" + }, + "execution_schedule.6.source_event [bridge]": { + "exception": "AssertionError", + "line": 111, + "result": "REJECTED", + "statement": "assert e==expected" + }, + "execution_schedule.6.src [bridge]": { + "exception": "AssertionError", + "line": 111, + "result": "REJECTED", + "statement": "assert e==expected" + }, + "execution_schedule.6.time [bridge]": { + "exception": "AssertionError", + "line": 112, + "result": "REJECTED", + "statement": "assert 0<=e['time']<144 and order[e['time']][1]==r" + }, + "expanded_scalar_interface": { + "result": "ACCEPTED" + }, + "hosting": { + "result": "ACCEPTED" + }, + "hosting.banks": { + "exception": "AssertionError", + "line": 96, + "result": "REJECTED", + "statement": "assert len(t['calls'])==144 and t['hosting']['max_helpers_per_physical_family_per_call']==1 and t['hosting']['banks']==[0]*9" + }, + "hosting.idle_remainder_semantics": { + "result": "ACCEPTED" + }, + "hosting.max_helpers_per_physical_family_per_call": { + "exception": "AssertionError", + "line": 96, + "result": "REJECTED", + "statement": "assert len(t['calls'])==144 and t['hosting']['max_helpers_per_physical_family_per_call']==1 and t['hosting']['banks']==[0]*9" + }, + "hosting.owner_map_sha256": { + "result": "ACCEPTED" + }, + "hosting.pooled_slots.0.child": { + "exception": "AssertionError", + "line": 172, + "result": "REJECTED", + "statement": "assert {k:z[k]for k in roles[key]}==roles[key]" + }, + "hosting.pooled_slots.0.copy": { + "exception": "AssertionError", + "line": 172, + "result": "REJECTED", + "statement": "assert {k:z[k]for k in roles[key]}==roles[key]" + }, + "hosting.pooled_slots.0.family": { + "exception": "AssertionError", + "line": 172, + "result": "REJECTED", + "statement": "assert {k:z[k]for k in roles[key]}==roles[key]" + }, + "hosting.pooled_slots.0.helper": { + "exception": "AssertionError", + "line": 172, + "result": "REJECTED", + "statement": "assert {k:z[k]for k in roles[key]}==roles[key]" + }, + "hosting.pooled_slots.0.offset": { + "exception": "AssertionError", + "line": 172, + "result": "REJECTED", + "statement": "assert {k:z[k]for k in roles[key]}==roles[key]" + }, + "hosting.pooled_slots.0.owner_replica": { + "exception": "AssertionError", + "line": 172, + "result": "REJECTED", + "statement": "assert {k:z[k]for k in roles[key]}==roles[key]" + }, + "hosting.pooled_slots.0.port": { + "exception": "AssertionError", + "line": 172, + "result": "REJECTED", + "statement": "assert {k:z[k]for k in roles[key]}==roles[key]" + }, + "hosting.pooled_slots.0.replica": { + "exception": "AssertionError", + "line": 172, + "result": "REJECTED", + "statement": "assert {k:z[k]for k in roles[key]}==roles[key]" + }, + "hosting.pooled_slots.0.stage": { + "exception": "AssertionError", + "line": 172, + "result": "REJECTED", + "statement": "assert {k:z[k]for k in roles[key]}==roles[key]" + }, + "hosting.pooled_slots.0.time": { + "result": "ACCEPTED" + }, + "hosting.pooled_slots.0.width": { + "exception": "AssertionError", + "line": 172, + "result": "REJECTED", + "statement": "assert {k:z[k]for k in roles[key]}==roles[key]" + }, + "hosting.rule": { + "result": "ACCEPTED" + }, + "hosting.scope": { + "result": "ACCEPTED" + }, + "idle_children": { + "exception": "AssertionError", + "line": 181, + "result": "REJECTED", + "statement": "assert t['idle_children']==wanted" + }, + "idle_children.0.child": { + "exception": "AssertionError", + "line": 181, + "result": "REJECTED", + "statement": "assert t['idle_children']==wanted" + }, + "idle_children.0.family": { + "exception": "AssertionError", + "line": 181, + "result": "REJECTED", + "statement": "assert t['idle_children']==wanted" + }, + "idle_children.0.hosted": { + "exception": "AssertionError", + "line": 181, + "result": "REJECTED", + "statement": "assert t['idle_children']==wanted" + }, + "idle_children.0.pending_offset": { + "exception": "AssertionError", + "line": 181, + "result": "REJECTED", + "statement": "assert t['idle_children']==wanted" + }, + "idle_children.0.port": { + "exception": "AssertionError", + "line": 181, + "result": "REJECTED", + "statement": "assert t['idle_children']==wanted" + }, + "idle_children.0.rank": { + "exception": "AssertionError", + "line": 181, + "result": "REJECTED", + "statement": "assert t['idle_children']==wanted" + }, + "idle_children.0.replica": { + "exception": "AssertionError", + "line": 181, + "result": "REJECTED", + "statement": "assert t['idle_children']==wanted" + }, + "idle_children.0.replicas": { + "exception": "AssertionError", + "line": 181, + "result": "REJECTED", + "statement": "assert t['idle_children']==wanted" + }, + "live": { + "exception": "AssertionError", + "line": 93, + "result": "REJECTED", + "statement": "assert(t['live'],t['physical'])==(DATA,PHYSICAL)" + }, + "local_providers": { + "exception": "AssertionError", + "line": 94, + "result": "REJECTED", + "statement": "assert t['local_providers']==registry and t['recipe']==parent['recipe']and 'each local invocation completes before the next' in t['loop_order'].lower()" + }, + "loop_order": { + "result": "ACCEPTED" + }, + "m": { + "result": "ACCEPTED" + }, + "physical": { + "exception": "AssertionError", + "line": 93, + "result": "REJECTED", + "statement": "assert(t['live'],t['physical'])==(DATA,PHYSICAL)" + }, + "port_restriction": { + "result": "ACCEPTED" + }, + "private_namespace": { + "exception": "AssertionError", + "line": 132, + "result": "REJECTED", + "statement": "assert t['private_namespace']=={'centers':[DATA,DATA+CENTRES],'reserved_tableau':[DATA+CENTRES,DATA+CENTRES+R],'coefficient':[DATA+CENTRES+R,PHYSICAL]}" + }, + "provider_dispatch": { + "result": "ACCEPTED" + }, + "recipe": { + "exception": "AssertionError", + "line": 94, + "result": "REJECTED", + "statement": "assert t['local_providers']==registry and t['recipe']==parent['recipe']and 'each local invocation completes before the next' in t['loop_order'].lower()" + }, + "replica_lift": { + "result": "ACCEPTED" + }, + "route_semantics": { + "result": "ACCEPTED" + }, + "supplier_substitution": { + "result": "ACCEPTED" + }, + "version": { + "result": "ACCEPTED" + } + }, + "new_mutants": { + "bank padding families restored": { + "exception": "AssertionError", + "line": 130, + "result": "REJECTED", + "statement": "assert t['bank_padding']==dict(parent['bank_padding'],families=[],children=[])" + }, + "bank padding global event moved to the front": { + "exception": "AssertionError", + "line": 122, + "result": "REJECTED", + "statement": "assert all('replica'in e for e in t['execution_schedule'][:-2])" + }, + "bridge coefficient negated": { + "exception": "AssertionError", + "line": 111, + "result": "REJECTED", + "statement": "assert e==expected" + }, + "bridge event dropped": { + "exception": "AssertionError", + "line": 120, + "result": "REJECTED", + "statement": "assert all(v==list(range(30))for v in perrep.values()) and globals_==[30,31]" + }, + "bridge retimed to another invocation of its replica": { + "exception": "AssertionError", + "line": 129, + "result": "REJECTED", + "statement": "assert e['time']==block" + }, + "call stage relabelled": { + "exception": "AssertionError", + "line": 103, + "result": "REJECTED", + "statement": "assert [list(x)for x in order]==t['async_schedule']['order'] and len(set(order))==144" + }, + "call time off by one": { + "exception": "AssertionError", + "line": 152, + "result": "REJECTED", + "statement": "s,rep,pair=call['stage'],call['replica'],call['pair'];assert call['time']==t['async_schedule']['order'].index([s,rep]);own=call['owners'];assert len(own)==N and" + }, + "data owner prefix permuted within a call": { + "exception": "AssertionError", + "line": 153, + "result": "REJECTED", + "statement": "assert own[:(2*V)]==old['owners'][:(2*V)]" + }, + "entering helper spectators permuted": { + "exception": "AssertionError", + "line": 159, + "result": "REJECTED", + "statement": "assert route==([3,i,off]if w==9 else[4,i,off,w,sp]);assert len(set(occ+sp))==9" + }, + "global events swapped": { + "exception": "AssertionError", + "line": 120, + "result": "REJECTED", + "statement": "assert all(v==list(range(30))for v in perrep.values()) and globals_==[30,31]" + }, + "global translation charge changed": { + "exception": "AssertionError", + "line": 119, + "result": "REJECTED", + "statement": "assert e==expected" + }, + "helper hosted after its idle child was swept (owner replica changed only)": { + "exception": "AssertionError", + "line": 156, + "result": "REJECTED", + "statement": "assert windows[ownerrep,i][0]= hi): + f2 = fam + (r2 - ownerrep) * 8 * V + if f2 not in fams[ci]: + return ci, h, f2 + raise AssertionError('no window mutant found') + + def set_family(ci, h, f2): + return lambda q: q['calls'][ci]['owners'][2 * V + h].__setitem__(0, f2) + + m = {} + m['helper hosted before its idle window opens (owner replica changed only)'] = set_family(*move_replica(True)) + m['helper hosted after its idle child was swept (owner replica changed only)'] = set_family(*move_replica(False)) + # Two width-nine helpers on one idle child of one owner replica: give the later one the earlier offset. + seen = {} + pick = None + for ci, h, fam, route, ownerrep, cp in helpers(): + if route[0] != 3: + continue + key = (route[1], ownerrep, fam % V) + if key in seen and seen[key][2] != route[2]: + pick = (ci, h, seen[key][2]) + break + seen.setdefault(key, (ci, h, route[2])) + assert pick + m['two helpers overlap axis cells of one idle child'] = \ + lambda q, p=pick: q['calls'][p[0]]['owners'][2 * V + p[1]][1].__setitem__(2, p[2]) + w9 = next((ci, h, route) for ci, h, fam, route, ownerrep, cp in helpers() if route[0] == 3) + m['width-nine helper runs past its idle child rank'] = \ + lambda q, p=w9: q['calls'][p[0]]['owners'][2 * V + p[1]][1].__setitem__(2, prank[p[2][1]] - 8) + ent = next((ci, h) for ci, h, fam, route, ownerrep, cp in helpers() if route[0] == 4 and len(route[4]) > 1) + m['entering helper spectators permuted'] = \ + lambda q, p=ent: q['calls'][p[0]]['owners'][2 * V + p[1]][1][4].reverse() + flip = lambda fam: fam + (V if (fam // V) % 2 == 0 else -V) # family = rep*8V + (2*pair+side)*V + port + side = next((ci, h, flip(fam)) for ci, h, fam, route, ownerrep, cp in helpers() if flip(fam) not in fams[ci]) + m['helper on the other side of its idle pair'] = set_family(*side) + m['data owner prefix permuted within a call'] = \ + lambda q: q['calls'][0]['owners'].__setitem__(slice(0, 2), [q['calls'][0]['owners'][1], q['calls'][0]['owners'][0]]) + m['call time off by one'] = lambda q: q['calls'][5].__setitem__('time', q['calls'][5]['time'] + 1) + m['call stage relabelled'] = lambda q: q['calls'][0].__setitem__('stage', 1) + m['pooled slot copy index changed'] = \ + lambda q: q['hosting']['pooled_slots'][0].__setitem__('copy', q['hosting']['pooled_slots'][0]['copy'] + 1) + m['pooled slot duplicated over another'] = \ + lambda q: q['hosting']['pooled_slots'].__setitem__(1, dict(q['hosting']['pooled_slots'][0])) + m['pooled slot time changed'] = \ + lambda q: q['hosting']['pooled_slots'][0].__setitem__('time', q['hosting']['pooled_slots'][0]['time'] + 1) + m['idle remainder rank plus one'] = lambda q: q['idle_children'][0].__setitem__('rank', q['idle_children'][0]['rank'] + 1) + m['idle remainder pending offset shifted'] = \ + lambda q: q['idle_children'][0].__setitem__('pending_offset', q['idle_children'][0]['pending_offset'] - 1) + bridge = next(k for k, e in enumerate(es) if e['kind'] == 'bridge') + m['bridge event dropped'] = lambda q, k=bridge: q['execution_schedule'].pop(k) + idle = next(k for k, e in enumerate(es) if e['kind'] == 'idle_cover_sweep') + m['idle sweep moved one position earlier'] = \ + lambda q, k=idle: q['execution_schedule'].insert(k - 1, q['execution_schedule'].pop(k)) + m['idle sweep levels changed'] = \ + lambda q, k=idle: q['execution_schedule'][k].__setitem__('to_level', q['execution_schedule'][k]['to_level'] + 1) + m['bridge coefficient negated'] = \ + lambda q, k=bridge: q['execution_schedule'][k].__setitem__('coefficient', -q['execution_schedule'][k]['coefficient']) + m['global translation charge changed'] = \ + lambda q: q['execution_schedule'][-1].__setitem__('charged_calls_per_cover', DATA - 1) + m['global events swapped'] = lambda q: q['execution_schedule'].__setitem__(slice(-2, None), q['execution_schedule'][-2:][::-1]) + m['bank padding families restored'] = lambda q: q['bank_padding'].__setitem__('families', [DATA, DATA]) + m['private namespace overlaps live data'] = \ + lambda q: q['private_namespace'].__setitem__('centers', [DATA - 1, q['private_namespace']['centers'][1]]) + m['two helpers per physical family declared'] = lambda q: q['hosting'].__setitem__('max_helpers_per_physical_family_per_call', 2) + m['one bank declared'] = lambda q: q['hosting']['banks'].__setitem__(0, 1) + m['source schedule tampered'] = \ + lambda q: q['async_schedule']['source_schedule'][0].__setitem__('stage', 3) + m['loop order sentence removed'] = lambda q: q.__setitem__('loop_order', 'Follow the schedule.') + m['local providers gain an extra entry'] = \ + lambda q: q['local_providers'].__setitem__('extra', q['local_providers'][next(iter(q['local_providers']))]) + # Targeted at assertions no earlier mutant reaches. + ends = {} + for ci, h, fam, route, ownerrep, cp in helpers(): + key = (route[1], ownerrep, fam % V) + w = 9 if route[0] == 3 else route[3] + if key not in ends or route[2] > ends[key][2]: + ends[key] = (ci, h, route[2], w, route[0]) + gap = next(v for k, v in sorted(ends.items()) if v[4] == 3 and v[2] + v[3] + 1 <= prank[k[0]]) + gc = ct['calls'][gap[0]] + gslot = next(k for k, z in enumerate(ct['hosting']['pooled_slots']) + if (z['stage'], z['replica'], z['helper']) == (gc['stage'], gc['replica'], gap[1])) + + def leave_gap(q, p=gap, k=gslot): # route and pooled slot move together, as one consistent emitter bug + q['calls'][p[0]]['owners'][2 * V + p[1]][1][2] = p[2] + 1 + q['hosting']['pooled_slots'][k]['offset'] = p[2] + 1 + m['hosted cells leave a gap in the pending prefix'] = leave_gap + order = [(c['stage'], c['replica']) for c in ct['calls']] + + def other_time(k): + r = es[k]['replica'] + return next(t for t, (s, rr) in enumerate(order) if rr == r and t != es[k]['time']) + loc = next(k for k, e in enumerate(es) if e['kind'] == 'local_cover_sweep') + m['local sweep retimed to another invocation of its replica'] = \ + lambda q, k=loc, t=other_time(loc): q['execution_schedule'][k].__setitem__('time', t) + m['bridge retimed to another invocation of its replica'] = \ + lambda q, k=bridge, t=other_time(bridge): q['execution_schedule'][k].__setitem__('time', t) + m['pooled slot dropped'] = lambda q: q['hosting']['pooled_slots'].pop() + m['pooled slot names a helper index past R'] = lambda q: q['hosting']['pooled_slots'][0].__setitem__('helper', R) + m['retired top-level local_programs reintroduced'] = lambda q: q.__setitem__('local_programs', {}) + l0, l1 = [k for k, e in enumerate(es) if e['kind'] == 'local_cover_sweep'][:2] + assert es[l0]['replica'] != es[l1]['replica'] + m['two local sweeps of different replicas swapped in list order'] = \ + lambda q, a=l0, b=l1: q['execution_schedule'].__setitem__(slice(a, b + 1), [q['execution_schedule'][b]] + q['execution_schedule'][a + 1:b] + [q['execution_schedule'][a]]) + m['bank padding global event moved to the front'] = \ + lambda q: q['execution_schedule'].insert(0, q['execution_schedule'].pop(-2)) + return m + + +def sentinel(v): + if isinstance(v, bool): + return not v + if isinstance(v, int): + return v + 1 + if isinstance(v, str): + return v + ' (mutated)' + if isinstance(v, list): + return v + [None] + if isinstance(v, dict): + return dict(v, mutated=True) + return 'mutated' + + +def census(G): + ct = G['ct'] + paths = [(k,) for k in ct] + paths += [('hosting', k) for k in ct['hosting'] if k != 'pooled_slots'] + paths += [('hosting', 'pooled_slots', 0, k) for k in ct['hosting']['pooled_slots'][0]] + paths += [('async_schedule', k) for k in ct['async_schedule']] + paths += [('calls', 0, k) for k in ct['calls'][0]] + paths += [('idle_children', 0, k) for k in ct['idle_children'][0]] + kinds = {} + for k, e in enumerate(ct['execution_schedule']): + kinds.setdefault(e['kind'], k) + for kind, k in sorted(kinds.items()): + paths += [('execution_schedule', k, f) for f in ct['execution_schedule'][k]] + out = {} + for p in paths: + def mut(q, p=p): + x = q + for s in p[:-1]: + x = x[s] + x[p[-1]] = sentinel(x[p[-1]]) + label = '.'.join(str(s) for s in p) + if p[0] == 'execution_schedule' and len(p) == 3: + label += ' [' + ct['execution_schedule'][p[1]]['kind'] + ']' + out[label] = mut + return out + + +def assert_lines(script): + tree = ast.parse(Path(script).read_text()) + fn = next(n for n in tree.body if isinstance(n, ast.FunctionDef) and n.name == 'audit') + return sorted({n.lineno for n in ast.walk(fn) if isinstance(n, ast.Assert)}) + + +def downstream_mentions(source, key): + hits = [] + for rel in ('code/invoice_allcold.py', 'code/assemble.py'): + text = (source / rel).read_text() + if "['%s']" % key in text or '"%s"' % key in text or "'%s'" % key in text: + hits.append(rel) + return hits + + +def main(): + ap = argparse.ArgumentParser() + ap.add_argument('--write', action='store_true', help='regenerate EXPECTED.json instead of comparing with it') + a = ap.parse_args() + with tempfile.TemporaryDirectory(prefix='t16-mutant-coverage-') as d: + tmp = Path(d) + source, script, g = extract(tmp) + G, archived = load_audit(source, script, g, tmp) + runner = Runner(G, script) + shipped = shipped_controls(G, runner, script) + assert list(shipped) == archived['mutations_rejected'], 'shipped control names differ from the selected receipt' + new = {k: runner.outcome(v) for k, v in new_mutants(G).items()} + # Cross-stage binding that no selected stage checks: the audited template's pooled owner map must be the + # frozen data/OWNERS.json.gz that lift16 and independent-t16 check, and owner_map_sha256 must name it. + owners_gz = (STAGE / 'data/OWNERS.json.gz').read_bytes() + frozen = json.loads(gzip.decompress(owners_gz)) + pooled = G['ct']['hosting']['pooled_slots'] + binding = dict(pooled_slots_equal_frozen_owners=pooled == frozen, + owner_map_sha256_names_frozen_gzip=G['ct']['hosting']['owner_map_sha256'] == sha(owners_gz), + roles=len(frozen)) + assert binding['pooled_slots_equal_frozen_owners'] and binding['owner_map_sha256_names_frozen_gzip'] + fields = {k: runner.outcome(v) for k, v in census(G).items()} + lines = assert_lines(script) + hit = defaultdict(list) + for group, res in (('shipped', shipped), ('new', new), ('field', fields)): + for name, o in res.items(): + if o.get('line'): + hit[o['line']].append(group + ': ' + name) + text = Path(script).read_text().splitlines() + coverage = {str(n): dict(statement=text[n - 1].strip()[:160], + shipped=sum(x.startswith('shipped') for x in hit.get(n, [])), + new=sum(x.startswith('new') for x in hit.get(n, [])), + field=sum(x.startswith('field') for x in hit.get(n, []))) + for n in lines} + accepted_fields = sorted(k for k, o in fields.items() if o['result'] == 'ACCEPTED') + result = dict( + status='PASS_T16_AUDIT_MUTANT_COVERAGE', + audit_script_sha256=AUDIT_SHA, + template_sha256=archived['template_sha256'], + template_to_frozen_owners_binding=binding, + shipped_controls=shipped, + new_mutants=new, + field_census=fields, + assert_lines_in_audit=len(lines), + assert_lines_reached_by_shipped=sorted(n for n in lines if coverage[str(n)]['shipped']), + assert_lines_reached_by_any=sorted(n for n in lines if any(coverage[str(n)][k] for k in ('shipped', 'new', 'field'))), + assert_lines_unreached=sorted(n for n in lines if not any(coverage[str(n)][k] for k in ('shipped', 'new', 'field'))), + coverage=coverage, + accepted_new_mutants=sorted(k for k, o in new.items() if o['result'] == 'ACCEPTED'), + accepted_fields={k: downstream_mentions(source, k.split(' ')[0].split('.')[-1]) for k in accepted_fields}, + rejections_by_exception=dict(Counter(o.get('exception', o['result']) for r in (shipped, new, fields) for o in r.values())), + ) + assert all(o['result'] == 'REJECTED' for o in shipped.values()), 'a shipped control is not rejected' + out = json.dumps(result, indent=1, sort_keys=True) + '\n' + if a.write: + (HERE / 'EXPECTED.json').write_text(out) + else: + expected = (HERE / 'EXPECTED.json').read_text() + assert out == expected, 'result differs from EXPECTED.json' + print('PASS_T16_AUDIT_MUTANT_COVERAGE shipped=%d new=%d fields=%d accepted_new=%d accepted_fields=%d unreached_asserts=%d' % ( + len(shipped), len(new), len(fields), len(result['accepted_new_mutants']), len(result['accepted_fields']), + len(result['assert_lines_unreached']))) + + +if __name__ == '__main__': + main()