From 5a1c98d8a434ee9e32c4c0cf6dccd223603c22ad Mon Sep 17 00:00:00 2001 From: Jade Lovelace Date: Mon, 14 Mar 2022 15:14:47 -0700 Subject: [PATCH] wip post --- .../posts/debugging-rr-children/.gitignore | 2 + content/posts/debugging-rr-children/Makefile | 5 ++ content/posts/debugging-rr-children/caller.c | 21 +++++ content/posts/debugging-rr-children/crasher.c | 7 ++ content/posts/debugging-rr-children/index.md | 80 +++++++++++++++++++ templates/macros/code.html | 36 +++++++++ templates/macros/colocated_asset.html | 10 +++ templates/shortcodes/codefile.html | 9 +++ 8 files changed, 170 insertions(+) create mode 100644 content/posts/debugging-rr-children/.gitignore create mode 100644 content/posts/debugging-rr-children/Makefile create mode 100644 content/posts/debugging-rr-children/caller.c create mode 100644 content/posts/debugging-rr-children/crasher.c create mode 100644 content/posts/debugging-rr-children/index.md create mode 100644 templates/macros/code.html create mode 100644 templates/macros/colocated_asset.html create mode 100644 templates/shortcodes/codefile.html diff --git a/content/posts/debugging-rr-children/.gitignore b/content/posts/debugging-rr-children/.gitignore new file mode 100644 index 0000000..d3802e6 --- /dev/null +++ b/content/posts/debugging-rr-children/.gitignore @@ -0,0 +1,2 @@ +/crasher +/caller diff --git a/content/posts/debugging-rr-children/Makefile b/content/posts/debugging-rr-children/Makefile new file mode 100644 index 0000000..9dab342 --- /dev/null +++ b/content/posts/debugging-rr-children/Makefile @@ -0,0 +1,5 @@ +CFLAGS = -g +all: caller crasher +.PHONY: all +caller: caller.o +crasher: crasher.o diff --git a/content/posts/debugging-rr-children/caller.c b/content/posts/debugging-rr-children/caller.c new file mode 100644 index 0000000..9bbc4c7 --- /dev/null +++ b/content/posts/debugging-rr-children/caller.c @@ -0,0 +1,21 @@ +#include +#include +#include + +int main(void) { + int pid = fork(); + if (pid == -1) { + perror("fork"); + return 1; + } else if (pid == 0) { + execl("./crasher", "./crasher", NULL); + return 1; + } else { + // parent + int status; + printf("[caller] spawned pid %d\n", pid); + int ret = waitpid(pid, &status, 0); + printf("[caller] waitpid: %d, exited? %d status %d, signaled? %d signal %d\n", ret, WIFEXITED(status), WEXITSTATUS(status), WIFSIGNALED(status), WTERMSIG(status)); + return 0; + } +} diff --git a/content/posts/debugging-rr-children/crasher.c b/content/posts/debugging-rr-children/crasher.c new file mode 100644 index 0000000..4979334 --- /dev/null +++ b/content/posts/debugging-rr-children/crasher.c @@ -0,0 +1,7 @@ +#include +#include + +int main(void) { + printf("[crasher] about to crash\n"); + abort(); +} diff --git a/content/posts/debugging-rr-children/index.md b/content/posts/debugging-rr-children/index.md new file mode 100644 index 0000000..6e1b11d --- /dev/null +++ b/content/posts/debugging-rr-children/index.md @@ -0,0 +1,80 @@ ++++ +date = "2022-02-24" +draft = false +path = "/blog/debugging-rr-children" +tags = [] +title = "Debugging: using rr to deal with unruly children (processes)" ++++ + +I have done multiple rounds of debugging blobs of processes that start together +and then something bad happens in one of the children several forks down. +Although gdb claims to support child processes with `set follow-fork-mode` +([docs][gdb-follow-fork-mode]), in practice, this is extremely painful since +you may have to set it to multiple different things in one reproduction. + +To deal with these, I've done such hacks as writing wrapper scripts for the +executable at fault that run it in a gdbserver. However, by far the worst one +I've done is printing out the PID of the misbehaving process then waiting, to +give me time to attach the debugger ([this is even suggested in the gdb +documentation][gdb-sleep]). + +[gdb-follow-fork-mode]: https://docs.jade.fyi/gnu/gdb/gdb.html#index-set-follow_002dfork_002dmode +[gdb-sleep]: https://docs.jade.fyi/gnu/gdb/gdb.html#Forks + +## Using rr to do it the easy way + +For this demo, I am using two programs I wrote: + +- `crasher` just prints out that it's about to crash, then aborts. +- `caller` forks and executes `crasher`, then prints its return value once it + exits. + +These are written in C but their source is not super interesting. Nevertheless, +you can find their source code [at the bottom of the post](#source). + +Here they are in action: + +``` +» ./caller +[caller] spawned pid 158938 +[crasher] about to crash +[caller] waitpid: 158938, exited? 0 status 0, signaled? 1 signal 6 +``` + +Signal 6, if you consult the table in `man 'signal(7)'`, is `SIGABRT` as +expected. + +We want to figure out why the crasher is crashing. It's possible to do with +gdb, but that's unnecessarily hard because of gdb, even moreso if it forks +multiple times. + +Let's use `rr` to do this more easily. First, record a run: + +``` +» rr record ./caller +rr: Saving execution to trace directory `/home/lf/.local/share/rr/caller-0'. +[caller] spawned pid 159146 +[crasher] about to crash +[caller] waitpid: 159146, exited? 0 status 0, signaled? 1 signal 6 +``` + +Then find the process ID of the crashing process: + +``` +» rr ps +PID PPID EXIT CMD +159145 -- 0 ./caller +159146 159145 -6 ./crasher +``` + +Next, use either `--onfork=` or `--onprocess=` to get a debugger on +the problem process: + +``` + + +## Demo source {#source} + +{{ codefile(path="caller.c", code_lang="c", colocated=true, hide=true) }} +{{ codefile(path="crasher.c", code_lang="c", colocated=true, hide=true) }} +{{ codefile(path="Makefile", code_lang="make", colocated=true, hide=true) }} diff --git a/templates/macros/code.html b/templates/macros/code.html new file mode 100644 index 0000000..d9c61ed --- /dev/null +++ b/templates/macros/code.html @@ -0,0 +1,36 @@ +{%- import "macros/colocated_asset.html" as colocated_asset -%} + + +{%- macro file(path, code_lang=false, colocated=false, + hide=false, show_path_with_prefix=false) -%} + + {%- set newline = " +" -%} + {%- set mypath = path -%} + + {%- if show_path_with_prefix == false -%} + {%- set header = "" -%} + {%- else -%} + {%- set header = show_path_with_prefix ~ " " ~ path ~ newline -%} + {%- endif -%} + {%- if colocated == true -%} + {%- set path = colocated_asset::colocated_asset(path=path) | trim -%} + {%- endif -%} + {%- if code_lang == true -%} + {%- set code_lang = '' -%} + {%- endif -%} + {%- set data = load_data(path=path, format="plain") -%} + {%- set source = "```" ~ code_lang ~ newline ~ header ~ data ~ newline ~ "```" | safe -%} + + + {%- if hide == true -%} +
+ + {{ mypath }} + + {%- endif -%} + {{ source | markdown(inline=true) | safe }} + {%- if hide == true -%} +
+ {%- endif -%} +{%- endmacro file -%} diff --git a/templates/macros/colocated_asset.html b/templates/macros/colocated_asset.html new file mode 100644 index 0000000..0cd131d --- /dev/null +++ b/templates/macros/colocated_asset.html @@ -0,0 +1,10 @@ + +{%- macro colocated_asset(path) -%} + {%- set page_url_components = page.relative_path | default(value=section.relative_path) | split(pat='/') -%} + {%- set page_base = page_url_components | slice(end=page_url_components | length - 1) | join(sep='/') -%} + {{ page_base ~ '/' ~ path }} +{%- endmacro colocated_asset -%} diff --git a/templates/shortcodes/codefile.html b/templates/shortcodes/codefile.html new file mode 100644 index 0000000..3269c19 --- /dev/null +++ b/templates/shortcodes/codefile.html @@ -0,0 +1,9 @@ +{%- import "macros/code.html" as code -%} + +{{ code::file( + path=path, + code_lang=code_lang | default(value=''), + colocated=colocated | default(value=false), + hide=hide | default(value=false), + show_path_with_prefix=show_path_with_prefix | default(value=false) +) }}