[{"data":1,"prerenderedAt":856},["ShallowReactive",2],{"\u002Fdeep-code-tracing":3},{"id":4,"title":5,"body":6,"createdAt":845,"description":12,"draft":846,"extension":847,"image":848,"mastodonThread":848,"meta":849,"navigation":603,"path":850,"seo":851,"stem":852,"tags":853,"updatedAt":854,"__hash__":855},"content\u002FDeep Code Tracing.md","Deep_Code_Tracing",{"type":7,"value":8,"toc":843},"minimark",[9,13,16,33,36,112,115,515,518,576,579,634,637,646,659,662,691,694,741,744,817,820,839],[10,11,12],"p",{},"Let's have some fun with tracing code!",[10,14,15],{},"Here is a nice way to get Elasticsearch+Kibana up and running:",[17,18,23],"pre",{"className":19,"code":20,"language":21,"meta":22,"style":22},"language-sh shiki shiki-themes github-light github-dark monokai","docker run -d -p 9200:9200 -p 5601:5601 nshou\u002Felasticsearch-kibana\n","sh","",[24,25,26],"code",{"__ignoreMap":22},[27,28,31],"span",{"class":29,"line":30},"line",1,[27,32,20],{},[10,34,35],{},"We're going to record our traces as a time-series. We unfortunately have to do a small config first, to ES know that our timestamp is ... a timestamp:",[17,37,39],{"className":19,"code":38,"language":21,"meta":22,"style":22},"curl -v -XPUT 'http:\u002F\u002Felastic:changeme@localhost:9200\u002Ftrace' -d '\n{\n  \"mappings\": {\n    \"trace\": {\n      \"properties\": {\n        \"timestamp\": {\n          \"type\": \"date\"\n        }\n      }\n    }\n  }\n}'\n",[24,40,41,46,52,58,64,70,76,82,88,94,100,106],{"__ignoreMap":22},[27,42,43],{"class":29,"line":30},[27,44,45],{},"curl -v -XPUT 'http:\u002F\u002Felastic:changeme@localhost:9200\u002Ftrace' -d '\n",[27,47,49],{"class":29,"line":48},2,[27,50,51],{},"{\n",[27,53,55],{"class":29,"line":54},3,[27,56,57],{},"  \"mappings\": {\n",[27,59,61],{"class":29,"line":60},4,[27,62,63],{},"    \"trace\": {\n",[27,65,67],{"class":29,"line":66},5,[27,68,69],{},"      \"properties\": {\n",[27,71,73],{"class":29,"line":72},6,[27,74,75],{},"        \"timestamp\": {\n",[27,77,79],{"class":29,"line":78},7,[27,80,81],{},"          \"type\": \"date\"\n",[27,83,85],{"class":29,"line":84},8,[27,86,87],{},"        }\n",[27,89,91],{"class":29,"line":90},9,[27,92,93],{},"      }\n",[27,95,97],{"class":29,"line":96},10,[27,98,99],{},"    }\n",[27,101,103],{"class":29,"line":102},11,[27,104,105],{},"  }\n",[27,107,109],{"class":29,"line":108},12,[27,110,111],{},"}'\n",[10,113,114],{},"Now we need to write down interesting things during our program execution. In my case, I'm working with a Rails project using RSpec. So I put this code near the top of spec\u002Fspec_helper.rb. Once executed, this code will start writing out to 'trace.log' lots and lots of tracing data:",[17,116,120],{"className":117,"code":118,"language":119,"meta":22,"style":22},"language-ruby shiki shiki-themes github-light github-dark monokai","git_sha = `git rev-parse HEAD`.chomp\ntrace_seq = 0\ntrace_log = File.new('trace.log', 'w')\ntrace_id = SecureRandom.uuid\ntrace = TracePoint.new(:line) do |tp|\n  next unless tp.path.start_with?(Rails.root.to_s)\n  event = {\n    trace_id: trace_id,\n    timestamp: Time.now.utc.round(10).iso8601(6),\n    git_sha: git_sha,\n    path: tp.path,\n    line: tp.lineno,\n    class: tp.defined_class.to_s,\n    method: tp.method_id,\n    seq: (trace_seq += 1)\n  }\n  if $rspec_example\n    event.merge!({\n      rspec_location: $rspec_example.location,\n      rspec_description: $rspec_example.description,\n      rspec_full_description: $rspec_example.full_description,\n    })\n  end\n  trace_log.puts(event.to_json)\nend\ntrace.enable\n","ruby",[24,121,122,144,155,186,202,230,266,276,284,326,334,346,358,375,388,405,410,419,431,445,458,471,477,483,500,506],{"__ignoreMap":22},[27,123,124,128,132,136,140],{"class":29,"line":30},[27,125,127],{"class":126},"sOrwc","git_sha",[27,129,131],{"class":130},"sC2Qs"," =",[27,133,135],{"class":134},"sstjo"," `git rev-parse HEAD`",[27,137,139],{"class":138},"sMOD_",".",[27,141,143],{"class":142},"sTrkL","chomp\n",[27,145,146,149,151],{"class":29,"line":48},[27,147,148],{"class":126},"trace_seq",[27,150,131],{"class":130},[27,152,154],{"class":153},"s7F3e"," 0\n",[27,156,157,160,162,166,168,171,174,177,180,183],{"class":29,"line":54},[27,158,159],{"class":126},"trace_log",[27,161,131],{"class":130},[27,163,165],{"class":164},"s-m8C"," File",[27,167,139],{"class":138},[27,169,170],{"class":130},"new",[27,172,173],{"class":138},"(",[27,175,176],{"class":134},"'trace.log'",[27,178,179],{"class":138},", ",[27,181,182],{"class":134},"'w'",[27,184,185],{"class":138},")\n",[27,187,188,191,193,196,198],{"class":29,"line":60},[27,189,190],{"class":126},"trace_id",[27,192,131],{"class":130},[27,194,195],{"class":164}," SecureRandom",[27,197,139],{"class":138},[27,199,201],{"class":200},"srTi1","uuid\n",[27,203,204,207,209,212,214,216,218,221,224,227],{"class":29,"line":66},[27,205,206],{"class":126},"trace",[27,208,131],{"class":130},[27,210,211],{"class":164}," TracePoint",[27,213,139],{"class":138},[27,215,170],{"class":130},[27,217,173],{"class":138},[27,219,220],{"class":153},":line",[27,222,223],{"class":138},") ",[27,225,226],{"class":130},"do",[27,228,229],{"class":138}," |tp|\n",[27,231,232,235,238,241,244,246,249,251,254,256,259,261,264],{"class":29,"line":72},[27,233,234],{"class":130},"  next",[27,236,237],{"class":130}," unless",[27,239,240],{"class":138}," tp.",[27,242,243],{"class":200},"path",[27,245,139],{"class":138},[27,247,248],{"class":200},"start_with?",[27,250,173],{"class":138},[27,252,253],{"class":164},"Rails",[27,255,139],{"class":138},[27,257,258],{"class":200},"root",[27,260,139],{"class":138},[27,262,263],{"class":200},"to_s",[27,265,185],{"class":138},[27,267,268,271,273],{"class":29,"line":78},[27,269,270],{"class":126},"  event",[27,272,131],{"class":130},[27,274,275],{"class":138}," {\n",[27,277,278,281],{"class":29,"line":84},[27,279,280],{"class":153},"    trace_id:",[27,282,283],{"class":138}," trace_id,\n",[27,285,286,289,292,294,297,299,302,304,307,309,312,315,318,320,323],{"class":29,"line":90},[27,287,288],{"class":153},"    timestamp:",[27,290,291],{"class":164}," Time",[27,293,139],{"class":138},[27,295,296],{"class":200},"now",[27,298,139],{"class":138},[27,300,301],{"class":200},"utc",[27,303,139],{"class":138},[27,305,306],{"class":200},"round",[27,308,173],{"class":138},[27,310,311],{"class":153},"10",[27,313,314],{"class":138},").",[27,316,317],{"class":200},"iso8601",[27,319,173],{"class":138},[27,321,322],{"class":153},"6",[27,324,325],{"class":138},"),\n",[27,327,328,331],{"class":29,"line":96},[27,329,330],{"class":153},"    git_sha:",[27,332,333],{"class":138}," git_sha,\n",[27,335,336,339,341,343],{"class":29,"line":102},[27,337,338],{"class":153},"    path:",[27,340,240],{"class":138},[27,342,243],{"class":200},[27,344,345],{"class":138},",\n",[27,347,348,351,353,356],{"class":29,"line":108},[27,349,350],{"class":153},"    line:",[27,352,240],{"class":138},[27,354,355],{"class":200},"lineno",[27,357,345],{"class":138},[27,359,361,364,366,369,371,373],{"class":29,"line":360},13,[27,362,363],{"class":153},"    class:",[27,365,240],{"class":138},[27,367,368],{"class":200},"defined_class",[27,370,139],{"class":138},[27,372,263],{"class":200},[27,374,345],{"class":138},[27,376,378,381,383,386],{"class":29,"line":377},14,[27,379,380],{"class":153},"    method:",[27,382,240],{"class":138},[27,384,385],{"class":200},"method_id",[27,387,345],{"class":138},[27,389,391,394,397,400,403],{"class":29,"line":390},15,[27,392,393],{"class":153},"    seq:",[27,395,396],{"class":138}," (trace_seq ",[27,398,399],{"class":130},"+=",[27,401,402],{"class":153}," 1",[27,404,185],{"class":138},[27,406,408],{"class":29,"line":407},16,[27,409,105],{"class":138},[27,411,413,416],{"class":29,"line":412},17,[27,414,415],{"class":130},"  if",[27,417,418],{"class":138}," $rspec_example\n",[27,420,422,425,428],{"class":29,"line":421},18,[27,423,424],{"class":138},"    event.",[27,426,427],{"class":200},"merge!",[27,429,430],{"class":138},"({\n",[27,432,434,437,440,443],{"class":29,"line":433},19,[27,435,436],{"class":153},"      rspec_location:",[27,438,439],{"class":138}," $rspec_example.",[27,441,442],{"class":200},"location",[27,444,345],{"class":138},[27,446,448,451,453,456],{"class":29,"line":447},20,[27,449,450],{"class":153},"      rspec_description:",[27,452,439],{"class":138},[27,454,455],{"class":200},"description",[27,457,345],{"class":138},[27,459,461,464,466,469],{"class":29,"line":460},21,[27,462,463],{"class":153},"      rspec_full_description:",[27,465,439],{"class":138},[27,467,468],{"class":200},"full_description",[27,470,345],{"class":138},[27,472,474],{"class":29,"line":473},22,[27,475,476],{"class":138},"    })\n",[27,478,480],{"class":29,"line":479},23,[27,481,482],{"class":130},"  end\n",[27,484,486,489,492,495,498],{"class":29,"line":485},24,[27,487,488],{"class":138},"  trace_log.",[27,490,491],{"class":142},"puts",[27,493,494],{"class":138},"(event.",[27,496,497],{"class":200},"to_json",[27,499,185],{"class":138},[27,501,503],{"class":29,"line":502},25,[27,504,505],{"class":130},"end\n",[27,507,509,512],{"class":29,"line":508},26,[27,510,511],{"class":138},"trace.",[27,513,514],{"class":200},"enable\n",[10,516,517],{},"I have some stuff in there that is specific to RSpec. Later in the same file I put this config to track the current test being executed:",[17,519,521],{"className":117,"code":520,"language":119,"meta":22,"style":22},"config.before :each do\n  $rspec_example = example\nend\nconfig.after :each do\n  $rspec_example = nil\nend\n",[24,522,523,537,548,552,563,572],{"__ignoreMap":22},[27,524,525,528,531,534],{"class":29,"line":30},[27,526,527],{"class":138},"config.",[27,529,530],{"class":200},"before",[27,532,533],{"class":153}," :each",[27,535,536],{"class":130}," do\n",[27,538,539,542,545],{"class":29,"line":48},[27,540,541],{"class":138},"  $rspec_example ",[27,543,544],{"class":130},"=",[27,546,547],{"class":138}," example\n",[27,549,550],{"class":29,"line":54},[27,551,505],{"class":130},[27,553,554,556,559,561],{"class":29,"line":60},[27,555,527],{"class":138},[27,557,558],{"class":200},"after",[27,560,533],{"class":153},[27,562,536],{"class":130},[27,564,565,567,569],{"class":29,"line":66},[27,566,541],{"class":138},[27,568,544],{"class":130},[27,570,571],{"class":153}," nil\n",[27,573,574],{"class":29,"line":72},[27,575,505],{"class":130},[10,577,578],{},"Then I run my tests, which outputs trace.log. With this tracing turned on, it went at about half the normal speed. Mine is 123M (after disabling some logging). Now we load it into our ES. I do this in batches of 10k, and then POST it into ES:",[17,580,582],{"className":19,"code":581,"language":21,"meta":22,"style":22},"cat trace.log \\\n  | jq -c '. | {\"index\": {\"_index\": \"trace\", \"_type\": \"trace\"}}, .' \\\n  | split -l 10000 - load-\n\nfor f in load-* ; do\n  cat $f | curl -XPOST 'http:\u002F\u002Felastic:changeme@localhost:9200\u002F_bulk' --data-binary @- >\u002Fdev\u002Fnull\ndone\n\n# I guess we could clean up now...\nrm load-*\n",[24,583,584,589,594,599,605,610,615,620,624,629],{"__ignoreMap":22},[27,585,586],{"class":29,"line":30},[27,587,588],{},"cat trace.log \\\n",[27,590,591],{"class":29,"line":48},[27,592,593],{},"  | jq -c '. | {\"index\": {\"_index\": \"trace\", \"_type\": \"trace\"}}, .' \\\n",[27,595,596],{"class":29,"line":54},[27,597,598],{},"  | split -l 10000 - load-\n",[27,600,601],{"class":29,"line":60},[27,602,604],{"emptyLinePlaceholder":603},true,"\n",[27,606,607],{"class":29,"line":66},[27,608,609],{},"for f in load-* ; do\n",[27,611,612],{"class":29,"line":72},[27,613,614],{},"  cat $f | curl -XPOST 'http:\u002F\u002Felastic:changeme@localhost:9200\u002F_bulk' --data-binary @- >\u002Fdev\u002Fnull\n",[27,616,617],{"class":29,"line":78},[27,618,619],{},"done\n",[27,621,622],{"class":29,"line":84},[27,623,604],{"emptyLinePlaceholder":603},[27,625,626],{"class":29,"line":90},[27,627,628],{},"# I guess we could clean up now...\n",[27,630,631],{"class":29,"line":96},[27,632,633],{},"rm load-*\n",[10,635,636],{},"OK! Now we can finally have some fun!",[10,638,639,640],{},"Load up ",[641,642,643],"a",{"href":643,"rel":644},"http:\u002F\u002Flocalhost:5601\u002F",[645],"nofollow",[647,648,649,653,656],"ul",{},[650,651,652],"li",{},"Set index name to \"trace\"",[650,654,655],{},"Accept the timestamp field it automatically picks",[650,657,658],{},"Click into discover, and now you can slice and dice some stuff!",[10,660,661],{},"Given a file, annotate the lines with a list of the tests that touch that line:",[17,663,665],{"className":19,"code":664,"language":21,"meta":22,"style":22},"FILEPATH=$1\n\ncurl -s -XGET \"localhost:9200\u002Ftrace\u002F_search?q=%2Bpath:$FILEPATH%20%2Bline:%5B1%20TO%20100000%5D&sort=timestamp:desc&size=10000&pretty\" \\\n  | jq -r '.hits.hits[]._source| \"\\(.line) \\(.rspec_location)\"' \\\n  | sort -u -n\n",[24,666,667,672,676,681,686],{"__ignoreMap":22},[27,668,669],{"class":29,"line":30},[27,670,671],{},"FILEPATH=$1\n",[27,673,674],{"class":29,"line":48},[27,675,604],{"emptyLinePlaceholder":603},[27,677,678],{"class":29,"line":54},[27,679,680],{},"curl -s -XGET \"localhost:9200\u002Ftrace\u002F_search?q=%2Bpath:$FILEPATH%20%2Bline:%5B1%20TO%20100000%5D&sort=timestamp:desc&size=10000&pretty\" \\\n",[27,682,683],{"class":29,"line":60},[27,684,685],{},"  | jq -r '.hits.hits[]._source| \"\\(.line) \\(.rspec_location)\"' \\\n",[27,687,688],{"class":29,"line":66},[27,689,690],{},"  | sort -u -n\n",[10,692,693],{},"Given a file and a line, list all of the tests that cover that line:",[17,695,697],{"className":19,"code":696,"language":21,"meta":22,"style":22},"#!\u002Fbin\u002Fsh\n\nFILEPATH=$1\nLINE=$2\n\ncurl -s -XGET \\\n  \"localhost:9200\u002Ftrace\u002F_search?q=%2Bpath:$FILEPATH%20%2Bline:$LINE&sort=timestamp:desc&size=1000&pretty\" \\\n  | jq -r '.hits.hits[]._source.rspec_location' \\\n  | sort -u\n",[24,698,699,704,708,712,717,721,726,731,736],{"__ignoreMap":22},[27,700,701],{"class":29,"line":30},[27,702,703],{},"#!\u002Fbin\u002Fsh\n",[27,705,706],{"class":29,"line":48},[27,707,604],{"emptyLinePlaceholder":603},[27,709,710],{"class":29,"line":54},[27,711,671],{},[27,713,714],{"class":29,"line":60},[27,715,716],{},"LINE=$2\n",[27,718,719],{"class":29,"line":66},[27,720,604],{"emptyLinePlaceholder":603},[27,722,723],{"class":29,"line":72},[27,724,725],{},"curl -s -XGET \\\n",[27,727,728],{"class":29,"line":78},[27,729,730],{},"  \"localhost:9200\u002Ftrace\u002F_search?q=%2Bpath:$FILEPATH%20%2Bline:$LINE&sort=timestamp:desc&size=1000&pretty\" \\\n",[27,732,733],{"class":29,"line":84},[27,734,735],{},"  | jq -r '.hits.hits[]._source.rspec_location' \\\n",[27,737,738],{"class":29,"line":90},[27,739,740],{},"  | sort -u\n",[10,742,743],{},"Ideas:",[647,745,746,769,772,786,789,803,806,814],{},[650,747,748,749],{},"Editor:\n",[647,750,751,754,757,760,763,766],{},[650,752,753],{},"Add to quickfix all of the tests that get here",[650,755,756],{},"Highlight on the screen in a heatmap way the code that leads to the current line and the code that leads away from the current line",[650,758,759],{},"Highlight untested lines",[650,761,762],{},"Given a test, highlight lines that get executed",[650,764,765],{},"Follow a trace within the editor, stepping forward\u002Fback\u002Fin\u002Fout. Choose your own adventure for where to step next",[650,767,768],{},"Highlight variables that are visible from the current scope",[650,770,771],{},"Given a test, generate a new copy of the codebase with only enough code to pass that one test",[650,773,774,775],{},"Visualize (graphviz) some interesting control flows\n",[647,776,777,780,783],{},[650,778,779],{},"Make an interactive (svg+web) trace",[650,781,782],{},"Step in, step over, switch sequences",[650,784,785],{},"Highlight past vs future in different colors",[650,787,788],{},"Record state in the trace -- local variable contents",[650,790,791,792],{},"Profiling:\n",[647,793,794,797,800],{},[650,795,796],{},"Block enter vs exit duration",[650,798,799],{},"Line duration",[650,801,802],{},"Variable scope & contents",[650,804,805],{},"Record during interactive execution (not only tests)",[650,807,808,809],{},"Capture line source code at time of trace?\n",[647,810,811],{},[650,812,813],{},"With cache to make it efficient",[650,815,816],{},"Extract into gem for recording, cli-app for using",[10,818,819],{},"Related work:",[647,821,822],{},[650,823,824,825],{},"Using Neo4j instead of ES\n",[647,826,827,833,836],{},[650,828,829],{},[641,830,831],{"href":831,"rel":832},"https:\u002F\u002Fneo4j.com\u002Fblog\u002Fneolytics-ruby-code-neo4j\u002F",[645],[650,834,835],{},"They also index a TON of other stuff -- AST, variable contents",[650,837,838],{},"Implementation is specific to Ruby",[840,841,842],"style",{},"html .default .shiki span {color: var(--shiki-default);background: var(--shiki-default-bg);font-style: var(--shiki-default-font-style);font-weight: var(--shiki-default-font-weight);text-decoration: var(--shiki-default-text-decoration);}html .shiki span {color: var(--shiki-default);background: var(--shiki-default-bg);font-style: var(--shiki-default-font-style);font-weight: var(--shiki-default-font-weight);text-decoration: var(--shiki-default-text-decoration);}html .dark .shiki span {color: var(--shiki-dark);background: var(--shiki-dark-bg);font-style: var(--shiki-dark-font-style);font-weight: var(--shiki-dark-font-weight);text-decoration: var(--shiki-dark-text-decoration);}html.dark .shiki span {color: var(--shiki-dark);background: var(--shiki-dark-bg);font-style: var(--shiki-dark-font-style);font-weight: var(--shiki-dark-font-weight);text-decoration: var(--shiki-dark-text-decoration);}html .sepia .shiki span {color: var(--shiki-sepia);background: var(--shiki-sepia-bg);font-style: var(--shiki-sepia-font-style);font-weight: var(--shiki-sepia-font-weight);text-decoration: var(--shiki-sepia-text-decoration);}html.sepia .shiki span {color: var(--shiki-sepia);background: var(--shiki-sepia-bg);font-style: var(--shiki-sepia-font-style);font-weight: var(--shiki-sepia-font-weight);text-decoration: var(--shiki-sepia-text-decoration);}html pre.shiki code .sOrwc, html code.shiki .sOrwc{--shiki-default:#E36209;--shiki-dark:#FFAB70;--shiki-sepia:#F8F8F2}html pre.shiki code .sC2Qs, html code.shiki .sC2Qs{--shiki-default:#D73A49;--shiki-dark:#F97583;--shiki-sepia:#F92672}html pre.shiki code .sstjo, html code.shiki .sstjo{--shiki-default:#032F62;--shiki-dark:#9ECBFF;--shiki-sepia:#E6DB74}html pre.shiki code .sMOD_, html code.shiki .sMOD_{--shiki-default:#24292E;--shiki-dark:#E1E4E8;--shiki-sepia:#F8F8F2}html pre.shiki code .sTrkL, html code.shiki .sTrkL{--shiki-default:#005CC5;--shiki-dark:#79B8FF;--shiki-sepia:#66D9EF}html pre.shiki code .s7F3e, html code.shiki .s7F3e{--shiki-default:#005CC5;--shiki-dark:#79B8FF;--shiki-sepia:#AE81FF}html pre.shiki code .s-m8C, html code.shiki .s-m8C{--shiki-default:#005CC5;--shiki-default-font-style:inherit;--shiki-dark:#79B8FF;--shiki-dark-font-style:inherit;--shiki-sepia:#66D9EF;--shiki-sepia-font-style:italic}html pre.shiki code .srTi1, html code.shiki .srTi1{--shiki-default:#6F42C1;--shiki-dark:#B392F0;--shiki-sepia:#A6E22E}",{"title":22,"searchDepth":48,"depth":48,"links":844},[],"2017-07-30T00:57-04:00",false,"md",null,{},"\u002Fdeep-code-tracing",{"title":5,"description":12},"Deep Code Tracing",[],"2017-07-31T10:29-04:00","U6bbm91Hk7iXzHieyQjMwBswh0_rrdrkeQSmUsFl8BA",1778988311328]