[analyzer] Add sm-malloc.dot

[David Malcolm <dmalcolm at redhat dot com>]

I've found myself wanting a visualization of the state machines,
so this patch adds a sm-malloc.dot, based on sm-malloc.cc

Ideally we would autogenerate the .dot file somehow, but there isn't a
good way to do this at this time.

A .png version of this .dot file can be seen at:
  https://dmalcolm.fedorapeople.org/gcc/2019-11-22/sm-malloc.png

Pushed to branch "dmalcolm/analyzer" on the GCC git mirror.

gcc/ChangeLog:
	* analyzer/sm-malloc.cc (class malloc_state_machine): Add note
	about sm-malloc.dot.
	(malloc_state_machine::on_stmt): Fix some comments to use "freed"
	rather than "free" as the name of a state.
	* analyzer/sm-malloc.dot: New file.
---
 gcc/analyzer/sm-malloc.cc  |  8 +++--
 gcc/analyzer/sm-malloc.dot | 89 ++++++++++++++++++++++++++++++++++++++++++++++
 2 files changed, 94 insertions(+), 3 deletions(-)
 create mode 100644 gcc/analyzer/sm-malloc.dot

diff --git a/gcc/analyzer/sm-malloc.cc b/gcc/analyzer/sm-malloc.cc
index 602cb0b..d3d9147 100644
--- a/gcc/analyzer/sm-malloc.cc
+++ b/gcc/analyzer/sm-malloc.cc
@@ -34,7 +34,9 @@ namespace {
 
 ////////////////////////////////////////////////////////////////////////////
 
-/* A state machine for detecting misuses of the malloc/free API.  */
+/* A state machine for detecting misuses of the malloc/free API.
+
+   See sm-malloc.dot for an overview (keep this in-sync with that file).  */
 
 class malloc_state_machine : public state_machine
 {
@@ -615,7 +617,7 @@ malloc_state_machine::on_stmt (sm_context *sm_ctxt,
 
 	  arg = sm_ctxt->get_readable_tree (arg);
 
-	  /* start/unchecked/nonnull -> free.  */
+	  /* start/unchecked/nonnull -> freed.  */
 	  sm_ctxt->on_transition (node, stmt, arg, m_start, m_freed);
 	  sm_ctxt->on_transition (node, stmt, arg, m_unchecked, m_freed);
 	  sm_ctxt->on_transition (node, stmt, arg, m_nonnull, m_freed);
@@ -623,7 +625,7 @@ malloc_state_machine::on_stmt (sm_context *sm_ctxt,
 	  /* Keep state "null" as-is, rather than transitioning to "free";
 	     we don't want want to complain about double-free of NULL.  */
 
-	  /* free -> stop, with warning.  */
+	  /* freed -> stop, with warning.  */
 	  sm_ctxt->warn_for_state (node, stmt, arg, m_freed,
 				   new double_free (*this, arg));
 	  sm_ctxt->on_transition (node, stmt, arg, m_freed, m_stop);
diff --git a/gcc/analyzer/sm-malloc.dot b/gcc/analyzer/sm-malloc.dot
new file mode 100644
index 0000000..2050267
--- /dev/null
+++ b/gcc/analyzer/sm-malloc.dot
@@ -0,0 +1,89 @@
+/* An overview of the state machine from sm-malloc.cc.
+   Copyright (C) 2019 Free Software Foundation, Inc.
+   Contributed by David Malcolm <dmalcolm@redhat.com>.
+
+This file is part of GCC.
+
+GCC is free software; you can redistribute it and/or modify it
+under the terms of the GNU General Public License as published by
+the Free Software Foundation; either version 3, or (at your option)
+any later version.
+
+GCC is distributed in the hope that it will be useful, but
+WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with GCC; see the file COPYING3.  If not see
+<http://www.gnu.org/licenses/>.  */
+
+/* Keep this in-sync with sm-malloc.cc  */
+
+digraph "malloc" {
+
+  /* STATES. */
+
+  /* Start state.  */
+  start;
+
+  /* State for a pointer returned from malloc that hasn't been checked for
+     NULL.
+     It could be a pointer to heap-allocated memory, or could be NULL.  */
+  unchecked;
+
+  /* State for a pointer that's known to be NULL.  */
+  null;
+
+  /* State for a pointer to heap-allocated memory, known to be non-NULL.  */
+  nonnull;
+
+  /* State for a pointer to freed memory.  */
+  freed;
+
+  /* State for a pointer that's known to not be on the heap (e.g. to a local
+     or global).  */
+  non_heap;
+
+  /* Stop state, for pointers we don't want to track any more.  */
+  stop;
+
+  /* TRANSITIONS. */
+
+  start -> unchecked [label="on 'X=malloc(...);'"];
+  start -> unchecked [label="on 'X=calloc(...);'"];
+
+  start -> non_heap [label="on 'X=alloca(...);'"];
+  start -> non_heap [label="on 'X=__builtin_alloca(...);'"];
+
+  /* On "free".  */
+  start -> freed [label="on 'free(X);'"];
+  unchecked -> freed [label="on 'free(X);'"];
+  nonnull -> freed [label="on 'free(X);'"];
+  freed -> stop [label="on 'free(X);':\n Warn('double-free')"];
+  non_heap -> stop  [label="on 'free(X);':\n Warn('free of non-heap')"];
+
+  /* Handle "__attribute__((nonnull))".   */
+  unchecked -> nonnull [label="on 'FN(X)' with __attribute__((nonnull)):\nWarn('possible NULL arg')"];
+  null -> stop [label="on 'FN(X)' with __attribute__((nonnull)):\nWarn('NULL arg')"];
+
+  /* is_zero_assignment.  */
+  start -> null [label="on 'X = 0;'"];
+  unchecked -> null [label="on 'X = 0;'"];
+  nonnull -> null [label="on 'X = 0;'"];
+  freed -> null [label="on 'X = 0;'"];
+
+  start -> non_heap [label="on 'X = &EXPR;'"];
+
+  /* Handle dereferences.  */
+  unchecked -> nonnull [label="on '*X':\nWarn('possible NULL deref')"];
+  null -> stop [label="on '*X':\nWarn('NULL deref')"];
+  freed -> stop [label="on '*X':\nWarn('use after free')"];
+
+  /* on_condition.  */
+  unchecked -> nonnull [label="on 'X != 0'"];
+  unchecked -> null [label="on 'X == 0'"];
+
+  unchecked -> stop [label="on leak:\nWarn('leak')"];
+  nonnull -> stop [label="on leak:\nWarn('leak')"];
+}
-- 
1.8.5.3