openmp: Add omp_all_memory support (C/C++ only so far)

The ugly part is that OpenMP 5.1 made omp_all_memory a reserved identifier
which isn't allowed to be used anywhere but in the depend clause, this is
against how everything else has been handled in OpenMP so far (where
some identifiers could have special meaning in some OpenMP clauses or
pragmas but not elsewhere).
The patch handles it by making it a conditional keyword (for -fopenmp
only) and emitting a better diagnostics when it is used in a primary
expression.  Having a nicer diagnostics when e.g. trying to do
int omp_all_memory;
or
int *omp_all_memory[10];
etc. would mean changing too many spots and hooking into name lookups
to reject declaring any such symbols would be too ugly and I'm afraid
there are way too many spots where one can introduce a name
(variables, functions, namespaces, struct, enum, enumerators, template
arguments, ...).

Otherwise, the handling is quite simple, normal depend clauses lower
into addresses of variables being handed over to the library, for
omp_all_memory I'm using NULL pointers.  omp_all_memory can only be
used with inout or out depend kinds and means that a task is dependent
on all previously created sibling tasks that have any dependency (of
any depend kind) and that any later created sibling tasks will be
dependent on it if they have any dependency.

2022-05-12  Jakub Jelinek  <jakub@redhat.com>

gcc/
	* gimplify.cc (gimplify_omp_depend): Don't build_fold_addr_expr
	if null_pointer_node.
	(gimplify_scan_omp_clauses): Likewise.
	* tree-pretty-print.cc (dump_omp_clause): Print null_pointer_node
	as omp_all_memory.
gcc/c-family/
	* c-common.h (enum rid): Add RID_OMP_ALL_MEMORY.
	* c-omp.cc (c_finish_omp_depobj): Don't build_fold_addr_expr
	if null_pointer_node.
gcc/c/
	* c-parser.cc (c_parse_init): Register omp_all_memory as keyword
	if flag_openmp.
	(c_parser_postfix_expression): Diagnose uses of omp_all_memory
	in postfix expressions.
	(c_parser_omp_variable_list): Handle omp_all_memory in depend
	clause.
	* c-typeck.cc (c_finish_omp_clauses): Handle omp_all_memory
	keyword in depend clause as null_pointer_node, diagnose invalid
	uses.
gcc/cp/
	* lex.cc (init_reswords): Register omp_all_memory as keyword
	if flag_openmp.
	* parser.cc (cp_parser_primary_expression): Diagnose uses of
	omp_all_memory in postfix expressions.
	(cp_parser_omp_var_list_no_open): Handle omp_all_memory in depend
	clause.
	* semantics.cc (finish_omp_clauses): Handle omp_all_memory
	keyword in depend clause as null_pointer_node, diagnose invalid
	uses.
	* pt.cc (tsubst_omp_clause_decl): Pass through omp_all_memory.
gcc/testsuite/
	* c-c++-common/gomp/all-memory-1.c: New test.
	* c-c++-common/gomp/all-memory-2.c: New test.
	* c-c++-common/gomp/all-memory-3.c: New test.
	* g++.dg/gomp/all-memory-1.C: New test.
	* g++.dg/gomp/all-memory-2.C: New test.
libgomp/
	* libgomp.h (struct gomp_task): Add depend_all_memory member.
	* task.c (gomp_init_task): Initialize depend_all_memory.
	(gomp_task_handle_depend): Handle omp_all_memory.
	(gomp_task_run_post_handle_depend_hash): Clear
	parent->depend_all_memory if equal to current task.
	(gomp_task_maybe_wait_for_dependencies): Handle omp_all_memory.
	* testsuite/libgomp.c-c++-common/depend-1.c: New test.
	* testsuite/libgomp.c-c++-common/depend-2.c: New test.
	* testsuite/libgomp.c-c++-common/depend-3.c: New test.

1 files changed, 167 insertions, 1 deletions

diff --git a/libgomp/task.c b/libgomp/task.c
index 828348c4cf4..db4a6f71fb7 100644
--- a/libgomp/task.c
+++ b/libgomp/task.c
@@ -80,6 +80,7 @@ gomp_init_task (struct gomp_task *task, struct gomp_task *parent_task,
   task->dependers = NULL;
   task->depend_hash = NULL;
   task->taskwait = NULL;
+  task->depend_all_memory = NULL;
   task->depend_count = 0;
   task->completion_sem = NULL;
   task->deferred_p = false;
@@ -171,6 +172,7 @@ gomp_task_handle_depend (struct gomp_task *task, struct gomp_task *parent,
   size_t ndepend = (uintptr_t) depend[0];
   size_t i;
   hash_entry_type ent;
+  bool all_memory = false;
 
   if (ndepend)
     {
@@ -181,6 +183,7 @@ gomp_task_handle_depend (struct gomp_task *task, struct gomp_task *parent,
 	{
 	  task->depend[i].addr = depend[2 + i];
 	  task->depend[i].is_in = i >= nout;
+	  all_memory |= i < nout && depend[2 + i] == NULL;
 	}
     }
   else
@@ -201,6 +204,8 @@ gomp_task_handle_depend (struct gomp_task *task, struct gomp_task *parent,
 	    {
 	    case GOMP_DEPEND_OUT:
 	    case GOMP_DEPEND_INOUT:
+	      all_memory |= d[0] == NULL;
+	      break;
 	    case GOMP_DEPEND_MUTEXINOUTSET:
 	      break;
 	    case GOMP_DEPEND_IN:
@@ -226,8 +231,126 @@ gomp_task_handle_depend (struct gomp_task *task, struct gomp_task *parent,
 	  task->depend[n++].is_in = 1;
 	}
     }
-  task->depend_count = ndepend;
   task->num_dependees = 0;
+  if (__builtin_expect (parent->depend_all_memory && ndepend, false))
+    {
+      struct gomp_task *tsk = parent->depend_all_memory;
+      if (tsk->dependers == NULL)
+	{
+	  tsk->dependers
+	    = gomp_malloc (sizeof (struct gomp_dependers_vec)
+			   + 6 * sizeof (struct gomp_task *));
+	  tsk->dependers->n_elem = 1;
+	  tsk->dependers->allocated = 6;
+	  tsk->dependers->elem[0] = task;
+	}
+      else
+	{
+	  if (tsk->dependers->n_elem == tsk->dependers->allocated)
+	    {
+	      tsk->dependers->allocated
+		= tsk->dependers->allocated * 2 + 2;
+	      tsk->dependers
+		= gomp_realloc (tsk->dependers,
+				sizeof (struct gomp_dependers_vec)
+				+ (tsk->dependers->allocated
+				   * sizeof (struct gomp_task *)));
+	    }
+	  tsk->dependers->elem[tsk->dependers->n_elem++] = task;
+	}
+      task->num_dependees++;
+    }
+  if (__builtin_expect (all_memory, false))
+    {
+      /* A task with depend(inout: omp_all_memory) depends on all previous
+	 sibling tasks which have any dependencies and all later sibling
+	 tasks which have any dependencies depend on it.  */
+      task->depend_count = 1;
+      task->depend[0].addr = NULL;
+      task->depend[0].next = NULL;
+      task->depend[0].prev = NULL;
+      task->depend[0].task = task;
+      task->depend[0].redundant = true;
+      task->depend[0].redundant_out = false;
+      if (parent->depend_hash)
+	{
+	  /* Inlined htab_traverse + htab_clear.  All newer siblings can
+	     just depend on this task.  Add dependencies on all previous
+	     sibling tasks with dependencies and make them redundant and
+	     clear the hash table.  */
+	  hash_entry_type *slot = &parent->depend_hash->entries[0];
+	  hash_entry_type *end = slot + htab_size (parent->depend_hash);
+	  for (; slot != end; ++slot)
+	    {
+	      if (*slot == HTAB_EMPTY_ENTRY)
+		continue;
+	      if (*slot != HTAB_DELETED_ENTRY)
+		{
+		  for (ent = *slot; ent; ent = ent->next)
+		    {
+		      struct gomp_task *tsk = ent->task;
+
+		      if (ent->redundant_out)
+			break;
+
+		      ent->redundant = true;
+		      if (tsk->dependers == NULL)
+			{
+			  tsk->dependers
+			    = gomp_malloc (sizeof (struct gomp_dependers_vec)
+					   + 6 * sizeof (struct gomp_task *));
+			  tsk->dependers->n_elem = 1;
+			  tsk->dependers->allocated = 6;
+			  tsk->dependers->elem[0] = task;
+			  task->num_dependees++;
+			  continue;
+			}
+		      /* We already have some other dependency on tsk from
+			 earlier depend clause.  */
+		      else if (tsk->dependers->n_elem
+			       && (tsk->dependers->elem[tsk->dependers->n_elem
+							- 1] == task))
+			continue;
+		      else if (tsk->dependers->n_elem
+			       == tsk->dependers->allocated)
+			{
+			  tsk->dependers->allocated
+			    = tsk->dependers->allocated * 2 + 2;
+			  tsk->dependers
+			    = gomp_realloc (tsk->dependers,
+					    sizeof (struct gomp_dependers_vec)
+					    + (tsk->dependers->allocated
+					       * sizeof (struct gomp_task *)));
+			}
+		      tsk->dependers->elem[tsk->dependers->n_elem++] = task;
+		      task->num_dependees++;
+		    }
+		  while (ent)
+		    {
+		      ent->redundant = true;
+		      ent = ent->next;
+		    }
+		}
+	      *slot = HTAB_EMPTY_ENTRY;
+	    }
+	  if (htab_size (parent->depend_hash) <= 32)
+	    {
+	      parent->depend_hash->n_elements = 0;
+	      parent->depend_hash->n_deleted = 0;
+	    }
+	  else
+	    {
+	      /* Shrink the hash table if it would be too large.
+		 We don't want to walk e.g. megabytes of empty hash
+		 table for every depend(inout: omp_all_memory).  */
+	      free (parent->depend_hash);
+	      parent->depend_hash = htab_create (12);
+	    }
+	}
+      parent->depend_all_memory = task;
+      return;
+    }
+  task->depend_count = ndepend;
   if (parent->depend_hash == NULL)
     parent->depend_hash = htab_create (2 * ndepend > 12 ? 2 * ndepend : 12);
   for (i = 0; i < ndepend; i++)
@@ -1175,6 +1298,8 @@ gomp_task_run_post_handle_depend_hash (struct gomp_task *child_task)
   struct gomp_task *parent = child_task->parent;
   size_t i;
 
+  if (parent->depend_all_memory == child_task)
+    parent->depend_all_memory = NULL;
   for (i = 0; i < child_task->depend_count; i++)
     if (!child_task->depend[i].redundant)
       {
@@ -1738,6 +1863,17 @@ gomp_task_maybe_wait_for_dependencies (void **depend)
       n = 5;
     }
   gomp_mutex_lock (&team->task_lock);
+  if (__builtin_expect (task->depend_all_memory && ndepend, false))
+    {
+      struct gomp_task *tsk = task->depend_all_memory;
+      if (!tsk->parent_depends_on)
+	{
+	  tsk->parent_depends_on = true;
+	  ++num_awaited;
+	  if (tsk->num_dependees == 0 && tsk->kind == GOMP_TASK_WAITING)
+	    priority_queue_upgrade_task (tsk, task);
+	}
+    }
   for (i = 0; i < ndepend; i++)
     {
       elem.addr = depend[i + n];
@@ -1760,6 +1896,36 @@ gomp_task_maybe_wait_for_dependencies (void **depend)
 	    }
 	  elem.addr = d[0];
 	}
+      if (__builtin_expect (elem.addr == NULL && !elem.is_in, false))
+	{
+	  size_t size = htab_size (task->depend_hash);
+	  if (htab_elements (task->depend_hash) * 8 < size && size > 32)
+	    htab_expand (task->depend_hash);
+
+	  /* depend(inout: omp_all_memory) - depend on all previous
+	     sibling tasks that do have dependencies.  Inlined
+	     htab_traverse.  */
+	  hash_entry_type *slot = &task->depend_hash->entries[0];
+	  hash_entry_type *end = slot + htab_size (task->depend_hash);
+	  for (; slot != end; ++slot)
+	    {
+	      if (*slot == HTAB_EMPTY_ENTRY || *slot == HTAB_DELETED_ENTRY)
+		continue;
+	      for (ent = *slot; ent; ent = ent->next)
+		{
+		  struct gomp_task *tsk = ent->task;
+		  if (!tsk->parent_depends_on)
+		    {
+		      tsk->parent_depends_on = true;
+		      ++num_awaited;
+		      if (tsk->num_dependees == 0
+			  && tsk->kind == GOMP_TASK_WAITING)
+			priority_queue_upgrade_task (tsk, task);
+		    }
+		}
+	    }
+	  break;
+	}
       ent = htab_find (task->depend_hash, &elem);
       for (; ent; ent = ent->next)
 	if (elem.is_in && ent->is_in)