Merge commit 'a857ec6aada08d8c334c389c7b682198d749b6c9^' into HEAD

ChangeLog

@@ -1,3 +1,23 @@
+2023-12-13  Arsen Arsenović  <arsen@aarsen.me>
+
+	* Makefile.def (gettext): Disable (via missing)
+	{install-,}{pdf,html,info,dvi} and TAGS targets.  Set no_install
+	to true.  Add --disable-threads --disable-libasprintf.  Drop the
+	lib_path (as there are no shared libs).
+	* Makefile.in: Regenerate.
+
+2023-12-12  Paul Iannetta  <piannetta@kalrayinc.com>
+
+	* MAINTAINERS: Add myself to write after approval
+
+2023-12-12  Feng Wang  <wangfeng@eswincomputing.com>
+
+	* MAINTAINERS: Update my email address
+
+2023-12-12  Feng Wang  <wangfeng@eswincomputing.com>
+
+	* MAINTAINERS: Add myself to write after approval
+
 2023-11-23  Nathaniel Shead  <nathanieloshead@gmail.com>
 
 	* MAINTAINERS: Add myself to write after approval and DCO

MAINTAINERS

@@ -472,6 +472,7 @@ Dominique d'Humieres				<dhumieres.dominique@free.fr>
 Andy Hutchinson					<hutchinsonandy@aim.com>
 Joel Hutton					<joel.hutton@arm.com>
 Lewis Hyatt					<lhyatt@gmail.com>
+Paul Iannetta					<piannetta@kalrayinc.com>
 Roland Illig					<roland.illig@gmx.de>
 Meador Inge					<meadori@codesourcery.com>
 Bernardo Innocenti				<bernie@develer.com>
@@ -706,6 +707,7 @@ Marcel Vollweiler				<marcel@codesourcery.com>
 Ville Voutilainen				<ville.voutilainen@gmail.com>
 Nenad Vukicevic					<nenad@intrepid.com>
 Feng Wang					<fengwang@nudt.edu.cn>
+Feng Wang					<wangfeng@eswincomputing.com>
 Hongyu Wang					<hongyu.wang@intel.com>
 Jiong Wang					<jiong.wang@arm.com>
 Stephen M. Webb					<stephen.webb@bregmasoft.com>

Makefile.def

@@ -80,8 +80,17 @@ host_modules= { module= gettext; bootstrap=true; no_install=true;
 		// need it in some configuratons, which is determined via nontrivial tests.
 		// Always enabling pic seems to make sense for something tied to
 		// user-facing output.
-                extra_configure_flags='--disable-shared --disable-java --disable-csharp --with-pic';
-                lib_path=intl/.libs; };
+		extra_configure_flags='--disable-shared --disable-threads --disable-java --disable-csharp --with-pic --disable-libasprintf';
+		missing= pdf;
+		missing= html;
+		missing= info;
+		missing= dvi;
+		missing= install-pdf;
+		missing= install-html;
+		missing= install-info;
+		missing= install-dvi;
+		missing= TAGS;
+		no_install= true; };
 host_modules= { module= tcl;
                 missing=mostlyclean; };
 host_modules= { module= itcl; };

Makefile.in

@@ -768,7 +768,7 @@ TARGET_LIB_PATH_libatomic = $$r/$(TARGET_SUBDIR)/libatomic/.libs:
 
 # This is the list of directories that may be needed in RPATH_ENVVAR
 # so that programs built for the host machine work.
-HOST_LIB_PATH = $(HOST_LIB_PATH_gmp)$(HOST_LIB_PATH_mpfr)$(HOST_LIB_PATH_mpc)$(HOST_LIB_PATH_isl)$(HOST_LIB_PATH_gettext)
+HOST_LIB_PATH = $(HOST_LIB_PATH_gmp)$(HOST_LIB_PATH_mpfr)$(HOST_LIB_PATH_mpc)$(HOST_LIB_PATH_isl)
 
 # Define HOST_LIB_PATH_gcc here, for the sake of TARGET_LIB_PATH, ouch
 @if gcc
@@ -796,11 +796,6 @@ HOST_LIB_PATH_isl = \
   $$r/$(HOST_SUBDIR)/isl/.libs:$$r/$(HOST_SUBDIR)/prev-isl/.libs:
 @endif isl
 
-@if gettext
-HOST_LIB_PATH_gettext = \
-  $$r/$(HOST_SUBDIR)/gettext/intl/.libs:$$r/$(HOST_SUBDIR)/prev-gettext/intl/.libs:
-@endif gettext
-
 
 CXX_FOR_TARGET_FLAG_TO_PASS = \
 	"CXX_FOR_TARGET=$(CXX_FOR_TARGET)"
@@ -19868,7 +19863,7 @@ configure-gettext:
 	  $$s/$$module_srcdir/configure \
 	  --srcdir=$${topdir}/$$module_srcdir \
 	  $(HOST_CONFIGARGS) --build=${build_alias} --host=${host_alias} \
-	  --target=${target_alias} --disable-shared --disable-java --disable-csharp --with-pic \
+	  --target=${target_alias} --disable-shared --disable-threads --disable-java --disable-csharp --with-pic --disable-libasprintf \
 	  || exit 1
 @endif gettext
 
@@ -19904,7 +19899,7 @@ configure-stage1-gettext:
 	  --target=${target_alias} \
 	   \
 	  $(STAGE1_CONFIGURE_FLAGS) \
-	  --disable-shared --disable-java --disable-csharp --with-pic
+	  --disable-shared --disable-threads --disable-java --disable-csharp --with-pic --disable-libasprintf
 @endif gettext-bootstrap
 
 .PHONY: configure-stage2-gettext maybe-configure-stage2-gettext
@@ -19938,7 +19933,7 @@ configure-stage2-gettext:
 	  --target=${target_alias} \
 	  --with-build-libsubdir=$(HOST_SUBDIR) \
 	  $(STAGE2_CONFIGURE_FLAGS) \
-	  --disable-shared --disable-java --disable-csharp --with-pic
+	  --disable-shared --disable-threads --disable-java --disable-csharp --with-pic --disable-libasprintf
 @endif gettext-bootstrap
 
 .PHONY: configure-stage3-gettext maybe-configure-stage3-gettext
@@ -19972,7 +19967,7 @@ configure-stage3-gettext:
 	  --target=${target_alias} \
 	  --with-build-libsubdir=$(HOST_SUBDIR) \
 	  $(STAGE3_CONFIGURE_FLAGS) \
-	  --disable-shared --disable-java --disable-csharp --with-pic
+	  --disable-shared --disable-threads --disable-java --disable-csharp --with-pic --disable-libasprintf
 @endif gettext-bootstrap
 
 .PHONY: configure-stage4-gettext maybe-configure-stage4-gettext
@@ -20006,7 +20001,7 @@ configure-stage4-gettext:
 	  --target=${target_alias} \
 	  --with-build-libsubdir=$(HOST_SUBDIR) \
 	  $(STAGE4_CONFIGURE_FLAGS) \
-	  --disable-shared --disable-java --disable-csharp --with-pic
+	  --disable-shared --disable-threads --disable-java --disable-csharp --with-pic --disable-libasprintf
 @endif gettext-bootstrap
 
 .PHONY: configure-stageprofile-gettext maybe-configure-stageprofile-gettext
@@ -20040,7 +20035,7 @@ configure-stageprofile-gettext:
 	  --target=${target_alias} \
 	  --with-build-libsubdir=$(HOST_SUBDIR) \
 	  $(STAGEprofile_CONFIGURE_FLAGS) \
-	  --disable-shared --disable-java --disable-csharp --with-pic
+	  --disable-shared --disable-threads --disable-java --disable-csharp --with-pic --disable-libasprintf
 @endif gettext-bootstrap
 
 .PHONY: configure-stagetrain-gettext maybe-configure-stagetrain-gettext
@@ -20074,7 +20069,7 @@ configure-stagetrain-gettext:
 	  --target=${target_alias} \
 	  --with-build-libsubdir=$(HOST_SUBDIR) \
 	  $(STAGEtrain_CONFIGURE_FLAGS) \
-	  --disable-shared --disable-java --disable-csharp --with-pic
+	  --disable-shared --disable-threads --disable-java --disable-csharp --with-pic --disable-libasprintf
 @endif gettext-bootstrap
 
 .PHONY: configure-stagefeedback-gettext maybe-configure-stagefeedback-gettext
@@ -20108,7 +20103,7 @@ configure-stagefeedback-gettext:
 	  --target=${target_alias} \
 	  --with-build-libsubdir=$(HOST_SUBDIR) \
 	  $(STAGEfeedback_CONFIGURE_FLAGS) \
-	  --disable-shared --disable-java --disable-csharp --with-pic
+	  --disable-shared --disable-threads --disable-java --disable-csharp --with-pic --disable-libasprintf
 @endif gettext-bootstrap
 
 .PHONY: configure-stageautoprofile-gettext maybe-configure-stageautoprofile-gettext
@@ -20142,7 +20137,7 @@ configure-stageautoprofile-gettext:
 	  --target=${target_alias} \
 	  --with-build-libsubdir=$(HOST_SUBDIR) \
 	  $(STAGEautoprofile_CONFIGURE_FLAGS) \
-	  --disable-shared --disable-java --disable-csharp --with-pic
+	  --disable-shared --disable-threads --disable-java --disable-csharp --with-pic --disable-libasprintf
 @endif gettext-bootstrap
 
 .PHONY: configure-stageautofeedback-gettext maybe-configure-stageautofeedback-gettext
@@ -20176,7 +20171,7 @@ configure-stageautofeedback-gettext:
 	  --target=${target_alias} \
 	  --with-build-libsubdir=$(HOST_SUBDIR) \
 	  $(STAGEautofeedback_CONFIGURE_FLAGS) \
-	  --disable-shared --disable-java --disable-csharp --with-pic
+	  --disable-shared --disable-threads --disable-java --disable-csharp --with-pic --disable-libasprintf
 @endif gettext-bootstrap
 
 
@@ -20633,23 +20628,8 @@ maybe-info-gettext:
 @if gettext
 maybe-info-gettext: info-gettext
 
-info-gettext: \
-    configure-gettext 
-	@[ -f ./gettext/Makefile ] || exit 0; \
-	r=`${PWD_COMMAND}`; export r; \
-	s=`cd $(srcdir); ${PWD_COMMAND}`; export s; \
-	$(HOST_EXPORTS) \
-	for flag in $(EXTRA_HOST_FLAGS) ; do \
-	  eval `echo "$$flag" | sed -e "s|^\([^=]*\)=\(.*\)|\1='\2'; export \1|"`; \
-	done; \
-	echo "Doing info in gettext"; \
-	(cd $(HOST_SUBDIR)/gettext && \
-	  $(MAKE) $(BASE_FLAGS_TO_PASS) "AR=$${AR}" "AS=$${AS}" \
-	          "CC=$${CC}" "CXX=$${CXX}" "LD=$${LD}" "NM=$${NM}" \
-	          "RANLIB=$${RANLIB}" \
-	          "DLLTOOL=$${DLLTOOL}" "WINDRES=$${WINDRES}" "WINDMC=$${WINDMC}" \
-	          info) \
-	  || exit 1
+# gettext doesn't support info.
+info-gettext:
 
 @endif gettext
 
@@ -20658,23 +20638,8 @@ maybe-dvi-gettext:
 @if gettext
 maybe-dvi-gettext: dvi-gettext
 
-dvi-gettext: \
-    configure-gettext 
-	@[ -f ./gettext/Makefile ] || exit 0; \
-	r=`${PWD_COMMAND}`; export r; \
-	s=`cd $(srcdir); ${PWD_COMMAND}`; export s; \
-	$(HOST_EXPORTS) \
-	for flag in $(EXTRA_HOST_FLAGS) ; do \
-	  eval `echo "$$flag" | sed -e "s|^\([^=]*\)=\(.*\)|\1='\2'; export \1|"`; \
-	done; \
-	echo "Doing dvi in gettext"; \
-	(cd $(HOST_SUBDIR)/gettext && \
-	  $(MAKE) $(BASE_FLAGS_TO_PASS) "AR=$${AR}" "AS=$${AS}" \
-	          "CC=$${CC}" "CXX=$${CXX}" "LD=$${LD}" "NM=$${NM}" \
-	          "RANLIB=$${RANLIB}" \
-	          "DLLTOOL=$${DLLTOOL}" "WINDRES=$${WINDRES}" "WINDMC=$${WINDMC}" \
-	          dvi) \
-	  || exit 1
+# gettext doesn't support dvi.
+dvi-gettext:
 
 @endif gettext
 
@@ -20683,23 +20648,8 @@ maybe-pdf-gettext:
 @if gettext
 maybe-pdf-gettext: pdf-gettext
 
-pdf-gettext: \
-    configure-gettext 
-	@[ -f ./gettext/Makefile ] || exit 0; \
-	r=`${PWD_COMMAND}`; export r; \
-	s=`cd $(srcdir); ${PWD_COMMAND}`; export s; \
-	$(HOST_EXPORTS) \
-	for flag in $(EXTRA_HOST_FLAGS) ; do \
-	  eval `echo "$$flag" | sed -e "s|^\([^=]*\)=\(.*\)|\1='\2'; export \1|"`; \
-	done; \
-	echo "Doing pdf in gettext"; \
-	(cd $(HOST_SUBDIR)/gettext && \
-	  $(MAKE) $(BASE_FLAGS_TO_PASS) "AR=$${AR}" "AS=$${AS}" \
-	          "CC=$${CC}" "CXX=$${CXX}" "LD=$${LD}" "NM=$${NM}" \
-	          "RANLIB=$${RANLIB}" \
-	          "DLLTOOL=$${DLLTOOL}" "WINDRES=$${WINDRES}" "WINDMC=$${WINDMC}" \
-	          pdf) \
-	  || exit 1
+# gettext doesn't support pdf.
+pdf-gettext:
 
 @endif gettext
 
@@ -20708,23 +20658,8 @@ maybe-html-gettext:
 @if gettext
 maybe-html-gettext: html-gettext
 
-html-gettext: \
-    configure-gettext 
-	@[ -f ./gettext/Makefile ] || exit 0; \
-	r=`${PWD_COMMAND}`; export r; \
-	s=`cd $(srcdir); ${PWD_COMMAND}`; export s; \
-	$(HOST_EXPORTS) \
-	for flag in $(EXTRA_HOST_FLAGS) ; do \
-	  eval `echo "$$flag" | sed -e "s|^\([^=]*\)=\(.*\)|\1='\2'; export \1|"`; \
-	done; \
-	echo "Doing html in gettext"; \
-	(cd $(HOST_SUBDIR)/gettext && \
-	  $(MAKE) $(BASE_FLAGS_TO_PASS) "AR=$${AR}" "AS=$${AS}" \
-	          "CC=$${CC}" "CXX=$${CXX}" "LD=$${LD}" "NM=$${NM}" \
-	          "RANLIB=$${RANLIB}" \
-	          "DLLTOOL=$${DLLTOOL}" "WINDRES=$${WINDRES}" "WINDMC=$${WINDMC}" \
-	          html) \
-	  || exit 1
+# gettext doesn't support html.
+html-gettext:
 
 @endif gettext
 
@@ -20733,23 +20668,8 @@ maybe-TAGS-gettext:
 @if gettext
 maybe-TAGS-gettext: TAGS-gettext
 
-TAGS-gettext: \
-    configure-gettext 
-	@[ -f ./gettext/Makefile ] || exit 0; \
-	r=`${PWD_COMMAND}`; export r; \
-	s=`cd $(srcdir); ${PWD_COMMAND}`; export s; \
-	$(HOST_EXPORTS) \
-	for flag in $(EXTRA_HOST_FLAGS) ; do \
-	  eval `echo "$$flag" | sed -e "s|^\([^=]*\)=\(.*\)|\1='\2'; export \1|"`; \
-	done; \
-	echo "Doing TAGS in gettext"; \
-	(cd $(HOST_SUBDIR)/gettext && \
-	  $(MAKE) $(BASE_FLAGS_TO_PASS) "AR=$${AR}" "AS=$${AS}" \
-	          "CC=$${CC}" "CXX=$${CXX}" "LD=$${LD}" "NM=$${NM}" \
-	          "RANLIB=$${RANLIB}" \
-	          "DLLTOOL=$${DLLTOOL}" "WINDRES=$${WINDRES}" "WINDMC=$${WINDMC}" \
-	          TAGS) \
-	  || exit 1
+# gettext doesn't support TAGS.
+TAGS-gettext:
 
 @endif gettext
 
@@ -20758,24 +20678,8 @@ maybe-install-info-gettext:
 @if gettext
 maybe-install-info-gettext: install-info-gettext
 
-install-info-gettext: \
-    configure-gettext \
-    info-gettext 
-	@[ -f ./gettext/Makefile ] || exit 0; \
-	r=`${PWD_COMMAND}`; export r; \
-	s=`cd $(srcdir); ${PWD_COMMAND}`; export s; \
-	$(HOST_EXPORTS) \
-	for flag in $(EXTRA_HOST_FLAGS) ; do \
-	  eval `echo "$$flag" | sed -e "s|^\([^=]*\)=\(.*\)|\1='\2'; export \1|"`; \
-	done; \
-	echo "Doing install-info in gettext"; \
-	(cd $(HOST_SUBDIR)/gettext && \
-	  $(MAKE) $(BASE_FLAGS_TO_PASS) "AR=$${AR}" "AS=$${AS}" \
-	          "CC=$${CC}" "CXX=$${CXX}" "LD=$${LD}" "NM=$${NM}" \
-	          "RANLIB=$${RANLIB}" \
-	          "DLLTOOL=$${DLLTOOL}" "WINDRES=$${WINDRES}" "WINDMC=$${WINDMC}" \
-	          install-info) \
-	  || exit 1
+# gettext doesn't support install-info.
+install-info-gettext:
 
 @endif gettext
 
@@ -20784,24 +20688,8 @@ maybe-install-dvi-gettext:
 @if gettext
 maybe-install-dvi-gettext: install-dvi-gettext
 
-install-dvi-gettext: \
-    configure-gettext \
-    dvi-gettext 
-	@[ -f ./gettext/Makefile ] || exit 0; \
-	r=`${PWD_COMMAND}`; export r; \
-	s=`cd $(srcdir); ${PWD_COMMAND}`; export s; \
-	$(HOST_EXPORTS) \
-	for flag in $(EXTRA_HOST_FLAGS) ; do \
-	  eval `echo "$$flag" | sed -e "s|^\([^=]*\)=\(.*\)|\1='\2'; export \1|"`; \
-	done; \
-	echo "Doing install-dvi in gettext"; \
-	(cd $(HOST_SUBDIR)/gettext && \
-	  $(MAKE) $(BASE_FLAGS_TO_PASS) "AR=$${AR}" "AS=$${AS}" \
-	          "CC=$${CC}" "CXX=$${CXX}" "LD=$${LD}" "NM=$${NM}" \
-	          "RANLIB=$${RANLIB}" \
-	          "DLLTOOL=$${DLLTOOL}" "WINDRES=$${WINDRES}" "WINDMC=$${WINDMC}" \
-	          install-dvi) \
-	  || exit 1
+# gettext doesn't support install-dvi.
+install-dvi-gettext:
 
 @endif gettext
 
@@ -20810,24 +20698,8 @@ maybe-install-pdf-gettext:
 @if gettext
 maybe-install-pdf-gettext: install-pdf-gettext
 
-install-pdf-gettext: \
-    configure-gettext \
-    pdf-gettext 
-	@[ -f ./gettext/Makefile ] || exit 0; \
-	r=`${PWD_COMMAND}`; export r; \
-	s=`cd $(srcdir); ${PWD_COMMAND}`; export s; \
-	$(HOST_EXPORTS) \
-	for flag in $(EXTRA_HOST_FLAGS) ; do \
-	  eval `echo "$$flag" | sed -e "s|^\([^=]*\)=\(.*\)|\1='\2'; export \1|"`; \
-	done; \
-	echo "Doing install-pdf in gettext"; \
-	(cd $(HOST_SUBDIR)/gettext && \
-	  $(MAKE) $(BASE_FLAGS_TO_PASS) "AR=$${AR}" "AS=$${AS}" \
-	          "CC=$${CC}" "CXX=$${CXX}" "LD=$${LD}" "NM=$${NM}" \
-	          "RANLIB=$${RANLIB}" \
-	          "DLLTOOL=$${DLLTOOL}" "WINDRES=$${WINDRES}" "WINDMC=$${WINDMC}" \
-	          install-pdf) \
-	  || exit 1
+# gettext doesn't support install-pdf.
+install-pdf-gettext:
 
 @endif gettext
 
@@ -20836,24 +20708,8 @@ maybe-install-html-gettext:
 @if gettext
 maybe-install-html-gettext: install-html-gettext
 
-install-html-gettext: \
-    configure-gettext \
-    html-gettext 
-	@[ -f ./gettext/Makefile ] || exit 0; \
-	r=`${PWD_COMMAND}`; export r; \
-	s=`cd $(srcdir); ${PWD_COMMAND}`; export s; \
-	$(HOST_EXPORTS) \
-	for flag in $(EXTRA_HOST_FLAGS) ; do \
-	  eval `echo "$$flag" | sed -e "s|^\([^=]*\)=\(.*\)|\1='\2'; export \1|"`; \
-	done; \
-	echo "Doing install-html in gettext"; \
-	(cd $(HOST_SUBDIR)/gettext && \
-	  $(MAKE) $(BASE_FLAGS_TO_PASS) "AR=$${AR}" "AS=$${AS}" \
-	          "CC=$${CC}" "CXX=$${CXX}" "LD=$${LD}" "NM=$${NM}" \
-	          "RANLIB=$${RANLIB}" \
-	          "DLLTOOL=$${DLLTOOL}" "WINDRES=$${WINDRES}" "WINDMC=$${WINDMC}" \
-	          install-html) \
-	  || exit 1
+# gettext doesn't support install-html.
+install-html-gettext:
 
 @endif gettext
 

contrib/ChangeLog

@@ -1,3 +1,10 @@
+2023-12-13  Arsen Arsenović  <arsen@aarsen.me>
+
+	* download_prerequisites
+	<arg parse>: Parse --only-gettext.
+	(echo_archives): Check only_gettext and stop early if true.
+	(helptext): Document --only-gettext.
+
 2023-12-02  Richard Sandiford  <richard.sandiford@arm.com>
 
 	* config-list.mk (OPT_IN_LANGUAGES): New variable.

contrib/download_prerequisites

@@ -36,16 +36,18 @@ gettext='gettext-0.22.tar.gz'
 base_url='http://gcc.gnu.org/pub/gcc/infrastructure/'
 
 echo_archives() {
+    echo "${gettext}"
+    if "${only_gettext}"; then return; fi
     echo "${gmp}"
     echo "${mpfr}"
     echo "${mpc}"
-    echo "${gettext}"
     if [ ${graphite} -gt 0 ]; then echo "${isl}"; fi
 }
 
 graphite=1
 verify=1
 force=0
+only_gettext=false
 OS=$(uname)
 
 if type wget > /dev/null ; then
@@ -74,6 +76,7 @@ The following options are available:
  --no-verify      don't verify package integrity
  --sha512         use SHA512 checksum to verify package integrity (default)
  --md5            use MD5 checksum to verify package integrity
+ --only-gettext   inhibit downloading any package but gettext
  --help           show this text and exit
  --version        show version information and exit
 "
@@ -159,6 +162,9 @@ do
                 chksum_extension='md5'
                 verify=1
                 ;;
+            --only-gettext)
+                only_gettext=true
+                ;;
             -*)
                 die "unknown option: ${arg}"
                 ;;

gcc/DATESTAMP

@@ -1 +1 @@
-20231206
+20231214

gcc/Makefile.in

@@ -1497,6 +1497,7 @@ OBJS = \
 	gimple-ssa-backprop.o \
 	gimple-ssa-isolate-paths.o \
 	gimple-ssa-nonnull-compare.o \
+	gimple-ssa-sccopy.o \
 	gimple-ssa-split-paths.o \
 	gimple-ssa-store-merging.o \
 	gimple-ssa-strength-reduction.o \
@@ -3735,7 +3736,6 @@ distclean: clean lang.distclean
 	-rm -f testsuite/*.log testsuite/*.sum
 	-cd testsuite && rm -f x *.x *.x? *.exe *.rpo *.o *.s *.S *.cc
 	-cd testsuite && rm -f *.out *.gcov *$(coverageexts)
-	-rm -rf ${QMTEST_DIR} stamp-qmtest
 	-rm -f .gdbinit configargs.h
 	-rm -f gcov.pod
 # Delete po/*.gmo only if we are not building in the source directory.
@@ -4417,58 +4417,6 @@ check-parallel-% : site.exp
 	  fi ; \
 	fi )
 
-# QMTest targets
-
-# The path to qmtest.
-QMTEST_PATH=qmtest
-
-# The flags to pass to qmtest.
-QMTESTFLAGS=
-
-# The flags to pass to "qmtest run".
-QMTESTRUNFLAGS=-f none --result-stream dejagnu_stream.DejaGNUStream
-
-# The command to use to invoke qmtest.
-QMTEST=${QMTEST_PATH} ${QMTESTFLAGS}
-
-# The tests (or suites) to run.
-QMTEST_GPP_TESTS=g++
-
-# The subdirectory of the OBJDIR that will be used to store the QMTest
-# test database configuration and that will be used for temporary
-# scratch space during QMTest's execution.
-QMTEST_DIR=qmtestsuite
-
-# Create the QMTest database configuration.
-${QMTEST_DIR} stamp-qmtest:
-	${QMTEST} -D ${QMTEST_DIR} create-tdb \
-	    -c gcc_database.GCCDatabase \
-	    -a srcdir=`cd ${srcdir}/testsuite && ${PWD_COMMAND}` && \
-	    $(STAMP) stamp-qmtest
-
-# Create the QMTest context file.
-${QMTEST_DIR}/context: stamp-qmtest
-	rm -f $@
-	echo "CompilerTable.languages=c cplusplus" >> $@
-	echo "CompilerTable.c_kind=GCC" >> $@
-	echo "CompilerTable.c_path=${objdir}/xgcc" >> $@
-	echo "CompilerTable.c_options=-B${objdir}/" >> $@
-	echo "CompilerTable.cplusplus_kind=GCC" >> $@
-	echo "CompilerTable.cplusplus_path=${objdir}/xg++" >> $@
-	echo "CompilerTable.cplusplus_options=-B${objdir}/" >> $@
-	echo "DejaGNUTest.target=${target_noncanonical}" >> $@
-
-# Run the G++ testsuite using QMTest.
-qmtest-g++: ${QMTEST_DIR}/context
-	cd ${QMTEST_DIR} && ${QMTEST} run ${QMTESTRUNFLAGS} -C context \
-	   -o g++.qmr ${QMTEST_GPP_TESTS}
-
-# Use the QMTest GUI.
-qmtest-gui: ${QMTEST_DIR}/context
-	cd ${QMTEST_DIR} && ${QMTEST} gui -C context
-
-.PHONY: qmtest-g++
-
 # Run Paranoia on real.cc.
 
 paranoia.o: $(srcdir)/../contrib/paranoia.cc $(CONFIG_H) $(SYSTEM_H) $(TREE_H)

gcc/ada/ChangeLog

@@ -1,3 +1,8 @@
+2023-12-11  Rainer Orth  <ro@CeBiTec.Uni-Bielefeld.DE>
+
+	* terminals.c [__FreeBSD__]: Include <libutil.h>.
+	(TABDLY): Only define if missing.
+
 2023-12-06  Alexandre Oliva  <oliva@adacore.com>
 
 	* gcc-interface/trans.cc: Include ipa-strub.h.

gcc/ada/terminals.c

@@ -1125,6 +1125,9 @@ __gnat_setup_winsize (void *desc ATTRIBUTE_UNUSED,
 #if defined (__APPLE__)
 #   include <util.h>
 #endif
+#if defined (__FreeBSD__)
+#   include <libutil.h>
+#endif
 
 #define CDISABLE _POSIX_VDISABLE
 
@@ -1265,11 +1268,13 @@ allocate_pty_desc (pty_desc **desc) {
 #ifndef NLDLY
 #define NLDLY 0
 #define CRDLY 0
-#define TABDLY 0
 #define BSDLY 0
 #define VTDLY 0
 #define FFDLY 0
 #endif
+#ifndef TABDLY
+#define TABDLY 0
+#endif
 
 /* child_setup_tty - set terminal properties
  *

gcc/analyzer/ChangeLog

@@ -1,3 +1,69 @@
+2023-12-11  David Malcolm  <dmalcolm@redhat.com>
+
+	PR analyzer/112955
+	* engine.cc (feasibility_state::feasibility_state): Initialize
+	m_snodes_visited.
+
+2023-12-11  Andrew Pinski  <apinski@marvell.com>
+
+	* region-model-manager.cc (maybe_undo_optimize_bit_field_compare): Remove
+	the check for type being unsigned_char_type_node.
+
+2023-12-08  David Malcolm  <dmalcolm@redhat.com>
+
+	* sm-taint.cc (taint_state_machine::alt_get_inherited_state): Fix
+	handling of TRUNC_MOD_EXPR.
+
+2023-12-08  David Malcolm  <dmalcolm@redhat.com>
+
+	* region-model.cc (contains_uninit_p): Only check for
+	svalues that the infoleak warning can handle.
+
+2023-12-08  David Malcolm  <dmalcolm@redhat.com>
+
+	PR analyzer/112889
+	* store.h (concrete_binding::concrete_binding): Strengthen
+	assertion to require size to be be positive, rather than just
+	non-zero.
+	(concrete_binding::mark_deleted): Use size rather than start bit
+	offset.
+	(concrete_binding::mark_empty): Likewise.
+	(concrete_binding::is_deleted): Likewise.
+	(concrete_binding::is_empty): Likewise.
+
+2023-12-07  Alexandre Oliva  <oliva@adacore.com>
+
+	* region-model.cc (has_nondefault_case_for_value_p): Take
+	enumerate type as a parameter.
+	(region_model::apply_constraints_for_gswitch): Cope with
+	integral promotion type casts.
+
+2023-12-07  David Malcolm  <dmalcolm@redhat.com>
+
+	PR analyzer/103546
+	PR analyzer/112850
+	* analyzer.opt (-param=analyzer-max-svalue-depth=): Increase from
+	12 to 18.
+	(Wanalyzer-symbol-too-complex): New.
+	* diagnostic-manager.cc
+	(null_assignment_sm_context::clear_all_per_svalue_state): New.
+	* engine.cc (impl_sm_context::clear_all_per_svalue_state): New.
+	* program-state.cc (sm_state_map::clear_all_per_svalue_state):
+	New.
+	* program-state.h (sm_state_map::clear_all_per_svalue_state): New
+	decl.
+	* region-model-manager.cc
+	(region_model_manager::reject_if_too_complex): Add
+	-Wanalyzer-symbol-too-complex.
+	* sm-taint.cc (taint_state_machine::on_condition): Handle
+	comparisons against UNKNOWN.
+	* sm.h (sm_context::clear_all_per_svalue_state): New.
+
+2023-12-06  David Malcolm  <dmalcolm@redhat.com>
+
+	* engine.cc (dump_analyzer_json): Use
+	flag_diagnostics_json_formatting.
+
 2023-12-01  David Malcolm  <dmalcolm@redhat.com>
 
 	* analyzer.h (class saved_diagnostic): New forward decl.

gcc/analyzer/analyzer.opt

@@ -43,7 +43,7 @@ Common Joined UInteger Var(param_analyzer_max_recursion_depth) Init(2) Param
 The maximum number of times a callsite can appear in a call stack within the analyzer, before terminating analysis of a call that would recurse deeper.
 
 -param=analyzer-max-svalue-depth=
-Common Joined UInteger Var(param_analyzer_max_svalue_depth) Init(12) Param
+Common Joined UInteger Var(param_analyzer_max_svalue_depth) Init(18) Param
 The maximum depth of a symbolic value, before approximating the value as unknown.
 
 -param=analyzer-min-snodes-for-call-summary=
@@ -262,6 +262,10 @@ Wanalyzer-use-of-uninitialized-value
 Common Var(warn_analyzer_use_of_uninitialized_value) Init(1) Warning
 Warn about code paths in which an uninitialized value is used.
 
+Wanalyzer-symbol-too-complex
+Common Var(warn_analyzer_symbol_too_complex) Init(0) Warning
+Warn if expressions are too complicated for the analyzer to fully track.
+
 Wanalyzer-too-complex
 Common Var(warn_analyzer_too_complex) Init(0) Warning
 Warn if the code is too complicated for the analyzer to fully explore.

gcc/analyzer/diagnostic-manager.cc

@@ -2043,6 +2043,11 @@ struct null_assignment_sm_context : public sm_context
     /* No-op.  */
   }
 
+  void clear_all_per_svalue_state () final override
+  {
+    /* No-op.  */
+  }
+
   void on_custom_transition (custom_transition *) final override
   {
   }

gcc/analyzer/engine.cc

@@ -474,6 +474,11 @@ public:
     m_new_state->m_checker_states[m_sm_idx]->set_global_state (state);
   }
 
+  void clear_all_per_svalue_state () final override
+  {
+    m_new_state->m_checker_states[m_sm_idx]->clear_all_per_svalue_state ();
+  }
+
   void on_custom_transition (custom_transition *transition) final override
   {
     transition->impl_transition (&m_eg,
@@ -4870,6 +4875,7 @@ feasibility_state::feasibility_state (const region_model &model,
 : m_model (model),
   m_snodes_visited (sg.m_nodes.length ())
 {
+  bitmap_clear (m_snodes_visited);
 }
 
 feasibility_state &
@@ -6068,7 +6074,7 @@ dump_analyzer_json (const supergraph &sg,
   toplev_obj->set ("egraph", eg.to_json ());
 
   pretty_printer pp;
-  toplev_obj->print (&pp);
+  toplev_obj->print (&pp, flag_diagnostics_json_formatting);
   pp_formatted_text (&pp);
 
   delete toplev_obj;

gcc/analyzer/program-state.cc

@@ -526,6 +526,14 @@ sm_state_map::clear_any_state (const svalue *sval)
   m_map.remove (sval);
 }
 
+/* Clear all per-svalue state within this state map.  */
+
+void
+sm_state_map::clear_all_per_svalue_state ()
+{
+  m_map.empty ();
+}
+
 /* Set the "global" state within this state map to STATE.  */
 
 void

gcc/analyzer/program-state.h

@@ -146,6 +146,7 @@ public:
 		       const svalue *origin,
 		       const extrinsic_state &ext_state);
   void clear_any_state (const svalue *sval);
+  void clear_all_per_svalue_state ();
 
   void set_global_state (state_machine::state_t state);
   state_machine::state_t get_global_state () const;

gcc/analyzer/region-model-manager.cc

@@ -185,6 +185,16 @@ region_model_manager::reject_if_too_complex (svalue *sval)
       return false;
     }
 
+  pretty_printer pp;
+  pp_format_decoder (&pp) = default_tree_printer;
+  sval->dump_to_pp (&pp, true);
+  if (warning_at (input_location, OPT_Wanalyzer_symbol_too_complex,
+		  "symbol too complicated: %qs",
+		  pp_formatted_text (&pp)))
+    inform (input_location,
+	    "max_depth %i exceeds --param=analyzer-max-svalue-depth=%i",
+	    c.m_max_depth, param_analyzer_max_svalue_depth);
+
   delete sval;
   return true;
 }
@@ -586,9 +596,6 @@ maybe_undo_optimize_bit_field_compare (tree type,
 				       tree cst,
 				       const svalue *arg1)
 {
-  if (type != unsigned_char_type_node)
-    return NULL;
-
   const binding_map &map = compound_sval->get_map ();
   unsigned HOST_WIDE_INT mask = TREE_INT_CST_LOW (cst);
   /* If "mask" is a contiguous range of set bits, see if the

gcc/analyzer/region-model.cc

@@ -5387,10 +5387,10 @@ has_nondefault_case_for_value_p (const gswitch *switch_stmt, tree int_cst)
    has nondefault cases handling all values in the enum.  */
 
 static bool
-has_nondefault_cases_for_all_enum_values_p (const gswitch *switch_stmt)
+has_nondefault_cases_for_all_enum_values_p (const gswitch *switch_stmt,
+					    tree type)
 {
   gcc_assert (switch_stmt);
-  tree type = TREE_TYPE (gimple_switch_index (switch_stmt));
   gcc_assert (TREE_CODE (type) == ENUMERAL_TYPE);
 
   for (tree enum_val_iter = TYPE_VALUES (type);
@@ -5426,6 +5426,23 @@ apply_constraints_for_gswitch (const switch_cfg_superedge &edge,
 {
   tree index  = gimple_switch_index (switch_stmt);
   const svalue *index_sval = get_rvalue (index, ctxt);
+  bool check_index_type = true;
+
+  /* With -fshort-enum, there may be a type cast.  */
+  if (ctxt && index_sval->get_kind () == SK_UNARYOP
+      && TREE_CODE (index_sval->get_type ()) == INTEGER_TYPE)
+    {
+      const unaryop_svalue *unaryop = as_a <const unaryop_svalue *> (index_sval);
+      if (unaryop->get_op () == NOP_EXPR
+	  && is_a <const initial_svalue *> (unaryop->get_arg ()))
+	if (const initial_svalue *initvalop = (as_a <const initial_svalue *>
+					       (unaryop->get_arg ())))
+	  if (TREE_CODE (initvalop->get_type ()) == ENUMERAL_TYPE)
+	    {
+	      index_sval = initvalop;
+	      check_index_type = false;
+	    }
+    }
 
   /* If we're switching based on an enum type, assume that the user is only
      working with values from the enum.  Hence if this is an
@@ -5437,12 +5454,14 @@ apply_constraints_for_gswitch (const switch_cfg_superedge &edge,
       ctxt
       /* Must be an enum value.  */
       && index_sval->get_type ()
-      && TREE_CODE (TREE_TYPE (index)) == ENUMERAL_TYPE
+      && (!check_index_type
+	  || TREE_CODE (TREE_TYPE (index)) == ENUMERAL_TYPE)
       && TREE_CODE (index_sval->get_type ()) == ENUMERAL_TYPE
       /* If we have a constant, then we can check it directly.  */
       && index_sval->get_kind () != SK_CONSTANT
       && edge.implicitly_created_default_p ()
-      && has_nondefault_cases_for_all_enum_values_p (switch_stmt)
+      && has_nondefault_cases_for_all_enum_values_p (switch_stmt,
+						     index_sval->get_type ())
       /* Don't do this if there's a chance that the index is
 	 attacker-controlled.  */
       && !ctxt->possibly_tainted_p (index_sval))
@@ -6557,22 +6576,33 @@ private:
 static bool
 contains_uninit_p (const svalue *sval)
 {
-  struct uninit_finder : public visitor
-  {
-  public:
-    uninit_finder () : m_found_uninit (false) {}
-    void visit_poisoned_svalue (const poisoned_svalue *sval)
+  switch (sval->get_kind ())
     {
-      if (sval->get_poison_kind () == POISON_KIND_UNINIT)
-	m_found_uninit = true;
+    default:
+      return false;
+    case SK_POISONED:
+      {
+	const poisoned_svalue *psval
+	  = as_a <const poisoned_svalue *> (sval);
+	return psval->get_poison_kind () == POISON_KIND_UNINIT;
+      }
+    case SK_COMPOUND:
+      {
+	const compound_svalue *compound_sval
+	  = as_a <const compound_svalue *> (sval);
+
+	for (auto iter : *compound_sval)
+	  {
+	    const svalue *sval = iter.second;
+	    if (const poisoned_svalue *psval
+		= sval->dyn_cast_poisoned_svalue ())
+	      if (psval->get_poison_kind () == POISON_KIND_UNINIT)
+		return true;
+	  }
+
+	return false;
+      }
     }
-    bool m_found_uninit;
-  };
-
-  uninit_finder v;
-  sval->accept (&v);
-
-  return v.m_found_uninit;
 }
 
 /* Function for use by plugins when simulating writing data through a

gcc/analyzer/sm-taint.cc

@@ -891,7 +891,6 @@ taint_state_machine::alt_get_inherited_state (const sm_state_map &map,
 	  case MULT_EXPR:
 	  case POINTER_PLUS_EXPR:
 	  case TRUNC_DIV_EXPR:
-	  case TRUNC_MOD_EXPR:
 	    {
 	      state_t arg0_state = map.get_state (arg0, ext_state);
 	      state_t arg1_state = map.get_state (arg1, ext_state);
@@ -899,6 +898,14 @@ taint_state_machine::alt_get_inherited_state (const sm_state_map &map,
 	    }
 	    break;
 
+	  case TRUNC_MOD_EXPR:
+	    {
+	      /* The left-hand side of X % Y can be sanitized by
+		 the operation.  */
+	      return map.get_state (arg1, ext_state);
+	    }
+	    break;
+
 	  case BIT_AND_EXPR:
 	  case RSHIFT_EXPR:
 	    return NULL;
@@ -1038,6 +1045,20 @@ taint_state_machine::on_condition (sm_context *sm_ctxt,
   if (stmt == NULL)
     return;
 
+  if (lhs->get_kind () == SK_UNKNOWN
+      || rhs->get_kind () == SK_UNKNOWN)
+    {
+      /* If we have a comparison against UNKNOWN, then
+	 we've presumably hit the svalue complexity limit,
+	 and we don't know what is being sanitized.
+	 Give up on any taint already found on this execution path.  */
+      // TODO: warn about this
+      if (get_logger ())
+	get_logger ()->log ("comparison against UNKNOWN; removing all taint");
+      sm_ctxt->clear_all_per_svalue_state ();
+      return;
+    }
+
   // TODO
   switch (op)
     {

gcc/analyzer/sm.h

@@ -299,6 +299,8 @@ public:
   virtual state_machine::state_t get_global_state () const = 0;
   virtual void set_global_state (state_machine::state_t) = 0;
 
+  virtual void clear_all_per_svalue_state () = 0;
+
   /* A vfunc for handling custom transitions, such as when registering
      a signal handler.  */
   virtual void on_custom_transition (custom_transition *transition) = 0;

gcc/analyzer/store.h

@@ -377,7 +377,7 @@ public:
   concrete_binding (bit_offset_t start_bit_offset, bit_size_t size_in_bits)
   : m_bit_range (start_bit_offset, size_in_bits)
   {
-    gcc_assert (!m_bit_range.empty_p ());
+    gcc_assert (m_bit_range.m_size_in_bits > 0);
   }
   bool concrete_p () const final override { return true; }
 
@@ -419,10 +419,10 @@ public:
 
   static int cmp_ptr_ptr (const void *, const void *);
 
-  void mark_deleted () { m_bit_range.m_start_bit_offset = -1; }
-  void mark_empty () { m_bit_range.m_start_bit_offset = -2; }
-  bool is_deleted () const { return m_bit_range.m_start_bit_offset == -1; }
-  bool is_empty () const { return m_bit_range.m_start_bit_offset == -2; }
+  void mark_deleted () { m_bit_range.m_size_in_bits = -1; }
+  void mark_empty () { m_bit_range.m_size_in_bits = -2; }
+  bool is_deleted () const { return m_bit_range.m_size_in_bits == -1; }
+  bool is_empty () const { return m_bit_range.m_size_in_bits == -2; }
 
 private:
   bit_range m_bit_range;

gcc/attribs.cc

@@ -315,7 +315,7 @@ void
 free_attr_data ()
 {
   for (auto x : ignored_attributes_table)
-    delete[] x;
+    delete x;
   ignored_attributes_table.release ();
 }
 
@@ -584,6 +584,19 @@ attribute_ignored_p (const attribute_spec *const as)
   return as->max_length == -2;
 }
 
+/* Return true if the ATTRS chain contains at least one attribute which
+   is not ignored.  */
+
+bool
+any_nonignored_attribute_p (tree attrs)
+{
+  for (tree attr = attrs; attr; attr = TREE_CHAIN (attr))
+    if (!attribute_ignored_p (attr))
+      return true;
+
+  return false;
+}
+
 /* See whether LIST contains at least one instance of attribute ATTR
    (possibly with different arguments).  Return the first such attribute
    if so, otherwise return null.  */

gcc/attribs.h

@@ -48,6 +48,7 @@ extern void apply_tm_attr (tree, tree);
 extern tree make_attribute (const char *, const char *, tree);
 extern bool attribute_ignored_p (tree);
 extern bool attribute_ignored_p (const attribute_spec *const);
+extern bool any_nonignored_attribute_p (tree);
 
 extern struct scoped_attributes *
   register_scoped_attributes (const scoped_attribute_specs &, bool = false);

gcc/builtin-types.def

@@ -840,6 +840,8 @@ DEF_FUNCTION_TYPE_4 (BT_FN_PTR_PTR_CONST_PTR_SIZE_SIZE,
 		     BT_PTR, BT_PTR, BT_CONST_PTR, BT_SIZE, BT_SIZE)
 DEF_FUNCTION_TYPE_4 (BT_FN_PTR_PTR_INT_SIZE_SIZE,
 		     BT_PTR, BT_PTR, BT_INT, BT_SIZE, BT_SIZE)
+DEF_FUNCTION_TYPE_4 (BT_FN_PTR_PTR_SIZE_PTRMODE_PTRMODE,
+		     BT_PTR, BT_PTR, BT_SIZE, BT_PTRMODE, BT_PTRMODE)
 DEF_FUNCTION_TYPE_4 (BT_FN_UINT_UINT_UINT_UINT_UINT,
 		     BT_UINT, BT_UINT, BT_UINT, BT_UINT, BT_UINT)
 DEF_FUNCTION_TYPE_4 (BT_FN_UINT_UINT_UINT_UINT_UINTPTR,

gcc/builtins.cc

@@ -1403,8 +1403,16 @@ get_memory_rtx (tree exp, tree len)
 
 /* Built-in functions to perform an untyped call and return.  */
 
+#define set_apply_args_size(x) \
+  (this_target_builtins->x_apply_args_size_plus_one = 1 + (x))
+#define get_apply_args_size() \
+  (this_target_builtins->x_apply_args_size_plus_one - 1)
 #define apply_args_mode \
   (this_target_builtins->x_apply_args_mode)
+#define set_apply_result_size(x) \
+  (this_target_builtins->x_apply_result_size_plus_one = 1 + (x))
+#define get_apply_result_size() \
+  (this_target_builtins->x_apply_result_size_plus_one - 1)
 #define apply_result_mode \
   (this_target_builtins->x_apply_result_mode)
 
@@ -1414,7 +1422,7 @@ get_memory_rtx (tree exp, tree len)
 static int
 apply_args_size (void)
 {
-  static int size = -1;
+  int size = get_apply_args_size ();
   int align;
   unsigned int regno;
 
@@ -1447,6 +1455,8 @@ apply_args_size (void)
 	  }
 	else
 	  apply_args_mode[regno] = as_a <fixed_size_mode> (VOIDmode);
+
+      set_apply_args_size (size);
     }
   return size;
 }
@@ -1457,7 +1467,7 @@ apply_args_size (void)
 static int
 apply_result_size (void)
 {
-  static int size = -1;
+  int size = get_apply_result_size ();
   int align, regno;
 
   /* The values computed by this function never change.  */
@@ -1489,6 +1499,8 @@ apply_result_size (void)
 #ifdef APPLY_RESULT_SIZE
       size = APPLY_RESULT_SIZE;
 #endif
+
+      set_apply_result_size (size);
     }
   return size;
 }
@@ -4284,6 +4296,40 @@ expand_builtin_memset (tree exp, rtx target, machine_mode mode)
   return expand_builtin_memset_args (dest, val, len, target, mode, exp);
 }
 
+/* Check that store_by_pieces allows BITS + LEN (so that we don't
+   expand something too unreasonably long), and every power of 2 in
+   BITS.  It is assumed that LEN has already been tested by
+   itself.  */
+static bool
+can_store_by_multiple_pieces (unsigned HOST_WIDE_INT bits,
+			      by_pieces_constfn constfun,
+			      void *constfundata, unsigned int align,
+			      bool memsetp,
+			      unsigned HOST_WIDE_INT len)
+{
+  if (bits
+      && !can_store_by_pieces (bits + len, constfun, constfundata,
+			       align, memsetp))
+    return false;
+
+  /* BITS set are expected to be generally in the low range and
+     contiguous.  We do NOT want to repeat the test above in case BITS
+     has a single bit set, so we terminate the loop when BITS == BIT.
+     In the unlikely case that BITS has the MSB set, also terminate in
+     case BIT gets shifted out.  */
+  for (unsigned HOST_WIDE_INT bit = 1; bit < bits && bit; bit <<= 1)
+    {
+      if ((bits & bit) == 0)
+	continue;
+
+      if (!can_store_by_pieces (bit, constfun, constfundata,
+				align, memsetp))
+	return false;
+    }
+
+  return true;
+}
+
 /* Try to store VAL (or, if NULL_RTX, VALC) in LEN bytes starting at TO.
    Return TRUE if successful, FALSE otherwise.  TO is assumed to be
    aligned at an ALIGN-bits boundary.  LEN must be a multiple of
@@ -4341,7 +4387,11 @@ try_store_by_multiple_pieces (rtx to, rtx len, unsigned int ctz_len,
   else
     /* Huh, max_len < min_len?  Punt.  See pr100843.c.  */
     return false;
-  if (min_len >= blksize)
+  if (min_len >= blksize
+      /* ??? Maybe try smaller fixed-prefix blksizes before
+	 punting?  */
+      && can_store_by_pieces (blksize, builtin_memset_read_str,
+			      &valc, align, true))
     {
       min_len -= blksize;
       min_bits = floor_log2 (min_len);
@@ -4367,8 +4417,9 @@ try_store_by_multiple_pieces (rtx to, rtx len, unsigned int ctz_len,
      happen because of the way max_bits and blksize are related, but
      it doesn't hurt to test.  */
   if (blksize > xlenest
-      || !can_store_by_pieces (xlenest, builtin_memset_read_str,
-			       &valc, align, true))
+      || !can_store_by_multiple_pieces (xlenest - blksize,
+					builtin_memset_read_str,
+					&valc, align, true, blksize))
     {
       if (!(flag_inline_stringops & ILSOP_MEMSET))
 	return false;
@@ -4386,17 +4437,17 @@ try_store_by_multiple_pieces (rtx to, rtx len, unsigned int ctz_len,
 	     of overflow.  */
 	  if (max_bits < orig_max_bits
 	      && xlenest + blksize >= xlenest
-	      && can_store_by_pieces (xlenest + blksize,
-				      builtin_memset_read_str,
-				      &valc, align, true))
+	      && can_store_by_multiple_pieces (xlenest,
+					       builtin_memset_read_str,
+					       &valc, align, true, blksize))
 	    {
 	      max_loop = true;
 	      break;
 	    }
 	  if (blksize
-	      && can_store_by_pieces (xlenest,
-				      builtin_memset_read_str,
-				      &valc, align, true))
+	      && can_store_by_multiple_pieces (xlenest,
+					       builtin_memset_read_str,
+					       &valc, align, true, 0))
 	    {
 	      max_len += blksize;
 	      min_len += blksize;
@@ -4519,7 +4570,7 @@ try_store_by_multiple_pieces (rtx to, rtx len, unsigned int ctz_len,
 	  to = change_address (to, QImode, 0);
 	  emit_move_insn (to, val);
 	  if (update_needed)
-	    next_ptr = plus_constant (ptr_mode, ptr, blksize);
+	    next_ptr = plus_constant (GET_MODE (ptr), ptr, blksize);
 	}
       else
 	{
@@ -12410,6 +12461,7 @@ builtin_fnspec (tree callee)
 	return ".cO ";
       /* Realloc serves both as allocation point and deallocation point.  */
       case BUILT_IN_REALLOC:
+      case BUILT_IN_GOMP_REALLOC:
 	return ".Cw ";
       case BUILT_IN_GAMMA_R:
       case BUILT_IN_GAMMAF_R:

gcc/builtins.h

@@ -37,6 +37,13 @@ struct target_builtins {
      register windows, this gives only the outbound registers.
      INCOMING_REGNO gives the corresponding inbound register.  */
   fixed_size_mode_pod x_apply_result_mode[FIRST_PSEUDO_REGISTER];
+
+  /* Nonzero iff the arrays above have been initialized.  The _plus_one suffix
+     is for zero initialization to make it an unreasonable size, used to signal
+     that the size and the corresponding mode array has not been
+     initialized.  */
+  int x_apply_args_size_plus_one;
+  int x_apply_result_size_plus_one;
 };
 
 extern struct target_builtins default_target_builtins;

gcc/c-family/ChangeLog

@@ -1,3 +1,69 @@
+2023-12-13  Patrick Palka  <ppalka@redhat.com>
+
+	* c.opt: Add -fdiagnostics-all-candidates.
+
+2023-12-13  Jason Merrill  <jason@redhat.com>
+
+	* c-warn.cc (check_address_or_pointer_of_packed_member):
+	Rename to check_address_of_packed_member.
+	(check_and_warn_address_or_pointer_of_packed_member):
+	Rename to check_and_warn_address_of_packed_member.
+	(warn_for_address_or_pointer_of_packed_member):
+	Rename to warn_for_address_of_packed_member.
+	* c-common.h: Adjust.
+
+2023-12-13  Jason Merrill  <jason@redhat.com>
+
+	* c-warn.cc (check_address_or_pointer_of_packed_member):
+	Remove warning based on TYPE_PACKED.
+
+2023-12-13  Julian Brown  <julian@codesourcery.com>
+
+	* c-common.h (c_omp_region_type): Add C_ORT_EXIT_DATA,
+	C_ORT_OMP_EXIT_DATA and C_ORT_ACC_TARGET.
+	(omp_addr_token): Add forward declaration.
+	(c_omp_address_inspector): New class.
+	* c-omp.cc (c_omp_adjust_map_clauses): Mark decls addressable here, but
+	do not change any mapping node types.
+	(c_omp_address_inspector::unconverted_ref_origin,
+	c_omp_address_inspector::component_access_p,
+	c_omp_address_inspector::check_clause,
+	c_omp_address_inspector::get_root_term,
+	c_omp_address_inspector::map_supported_p,
+	c_omp_address_inspector::get_origin,
+	c_omp_address_inspector::maybe_unconvert_ref,
+	c_omp_address_inspector::maybe_zero_length_array_section,
+	c_omp_address_inspector::expand_array_base,
+	c_omp_address_inspector::expand_component_selector,
+	c_omp_address_inspector::expand_map_clause): New methods.
+	(omp_expand_access_chain): New function.
+
+2023-12-12  Richard Biener  <rguenther@suse.de>
+
+	PR ipa/92606
+	* c-attribs.cc (handle_noicf_attribute): Also allow the
+	attribute on global variables.
+
+2023-12-10  Ken Matsui  <kmatsui@gcc.gnu.org>
+	    Patrick Palka  <ppalka@redhat.com>
+
+	* c-common.cc (c_common_reswords): Remove all mappings of
+	built-in traits.
+	* c-common.h (enum rid): Remove all RID values for built-in
+	traits.
+
+2023-12-07  Andrew Pinski  <pinskia@gmail.com>
+	    Jakub Jelinek  <jakub@redhat.com>
+
+	PR preprocessor/111965
+	* c-opts.cc (c_common_handle_option) <case OPT_fdebug_cpp>: Set
+	cpp_opts->debug to value rather than 1.
+
+2023-12-06  David Malcolm  <dmalcolm@redhat.com>
+
+	* c-opts.cc (c_diagnostic_finalizer): Make "diagnostic" param
+	const.
+
 2023-12-06  Alexandre Oliva  <oliva@adacore.com>
 
 	* c-attribs.cc: Include ipa-strub.h.

gcc/c-family/c-attribs.cc

@@ -1649,7 +1649,8 @@ handle_noicf_attribute (tree *node, tree name,
 			tree ARG_UNUSED (args),
 			int ARG_UNUSED (flags), bool *no_add_attrs)
 {
-  if (TREE_CODE (*node) != FUNCTION_DECL)
+  if (TREE_CODE (*node) != FUNCTION_DECL
+      && (TREE_CODE (*node) != VAR_DECL || !is_global_var (*node)))
     {
       warning (OPT_Wattributes, "%qE attribute ignored", name);
       *no_add_attrs = true;

gcc/c-family/c-common.cc

@@ -560,13 +560,6 @@ const struct c_common_resword c_common_reswords[] =
   { "wchar_t",		RID_WCHAR,	D_CXXONLY },
   { "while",		RID_WHILE,	0 },
 
-#define DEFTRAIT(TCC, CODE, NAME, ARITY) \
-  { NAME,		RID_##CODE,	D_CXXONLY },
-#include "cp/cp-trait.def"
-#undef DEFTRAIT
-  /* An alias for __is_same.  */
-  { "__is_same_as",	RID_IS_SAME,	D_CXXONLY },
-
   /* C++ transactional memory.  */
   { "synchronized",	RID_SYNCHRONIZED, D_CXX_OBJC | D_TRANSMEM },
   { "atomic_noexcept",	RID_ATOMIC_NOEXCEPT, D_CXXONLY | D_TRANSMEM },

gcc/c-family/c-common.h

@@ -168,11 +168,6 @@ enum rid
   RID_BUILTIN_LAUNDER,
   RID_BUILTIN_BIT_CAST,
 
-#define DEFTRAIT(TCC, CODE, NAME, ARITY) \
-  RID_##CODE,
-#include "cp/cp-trait.def"
-#undef DEFTRAIT
-
   /* C++11 */
   RID_CONSTEXPR, RID_DECLTYPE, RID_NOEXCEPT, RID_NULLPTR, RID_STATIC_ASSERT,
 
@@ -1284,8 +1279,11 @@ enum c_omp_region_type
   C_ORT_ACC			= 1 << 1,
   C_ORT_DECLARE_SIMD		= 1 << 2,
   C_ORT_TARGET			= 1 << 3,
+  C_ORT_EXIT_DATA		= 1 << 4,
   C_ORT_OMP_DECLARE_SIMD	= C_ORT_OMP | C_ORT_DECLARE_SIMD,
-  C_ORT_OMP_TARGET		= C_ORT_OMP | C_ORT_TARGET
+  C_ORT_OMP_TARGET		= C_ORT_OMP | C_ORT_TARGET,
+  C_ORT_OMP_EXIT_DATA		= C_ORT_OMP | C_ORT_EXIT_DATA,
+  C_ORT_ACC_TARGET		= C_ORT_ACC | C_ORT_TARGET
 };
 
 extern tree c_finish_omp_master (location_t, tree);
@@ -1322,6 +1320,72 @@ extern tree c_omp_check_context_selector (location_t, tree);
 extern void c_omp_mark_declare_variant (location_t, tree, tree);
 extern void c_omp_adjust_map_clauses (tree, bool);
 
+namespace omp_addr_tokenizer { struct omp_addr_token; }
+typedef omp_addr_tokenizer::omp_addr_token omp_addr_token;
+
+class c_omp_address_inspector
+{
+  location_t loc;
+  tree root_term;
+  bool indirections;
+  int map_supported;
+
+protected:
+  tree orig;
+
+public:
+  c_omp_address_inspector (location_t loc, tree t)
+    : loc (loc), root_term (NULL_TREE), indirections (false),
+      map_supported (-1), orig (t)
+    {
+    }
+
+  ~c_omp_address_inspector ()
+    {
+    }
+
+  virtual bool processing_template_decl_p ()
+    {
+      return false;
+    }
+
+  virtual void emit_unmappable_type_notes (tree)
+    {
+    }
+
+  virtual tree convert_from_reference (tree)
+    {
+      gcc_unreachable ();
+    }
+
+  virtual tree build_array_ref (location_t loc, tree arr, tree idx)
+    {
+      tree eltype = TREE_TYPE (TREE_TYPE (arr));
+      return build4_loc (loc, ARRAY_REF, eltype, arr, idx, NULL_TREE,
+			 NULL_TREE);
+    }
+
+  virtual bool check_clause (tree);
+  tree get_root_term (bool);
+
+  tree unconverted_ref_origin ();
+  bool component_access_p ();
+
+  bool map_supported_p ();
+
+  static tree get_origin (tree);
+  static tree maybe_unconvert_ref (tree);
+
+  bool maybe_zero_length_array_section (tree);
+
+  tree expand_array_base (tree, vec<omp_addr_token *> &, tree, unsigned *,
+			  c_omp_region_type);
+  tree expand_component_selector (tree, vec<omp_addr_token *> &, tree,
+				  unsigned *, c_omp_region_type);
+  tree expand_map_clause (tree, tree, vec<omp_addr_token *> &,
+			  c_omp_region_type);
+};
+
 enum c_omp_directive_kind {
   C_OMP_DIR_STANDALONE,
   C_OMP_DIR_CONSTRUCT,
@@ -1508,7 +1572,7 @@ extern void warnings_for_convert_and_check (location_t, tree, tree, tree);
 extern void c_do_switch_warnings (splay_tree, location_t, tree, tree, bool);
 extern void warn_for_omitted_condop (location_t, tree);
 extern bool warn_for_restrict (unsigned, tree *, unsigned);
-extern void warn_for_address_or_pointer_of_packed_member (tree, tree);
+extern void warn_for_address_of_packed_member (tree, tree);
 extern void warn_parm_array_mismatch (location_t, tree, tree);
 extern void maybe_warn_sizeof_array_div (location_t, tree, tree, tree, tree);
 extern void do_warn_array_compare (location_t, tree_code, tree, tree);

gcc/c-family/c-omp.cc

@@ -3169,8 +3169,9 @@ struct map_clause
     decl_mapped (false), omp_declare_target (false) { }
 };
 
-/* Adjust map clauses after normal clause parsing, mainly to turn specific
-   base-pointer map cases into attach/detach and mark them addressable.  */
+/* Adjust map clauses after normal clause parsing, mainly to mark specific
+   base-pointer map cases addressable that may be turned into attach/detach
+   operations during gimplification.  */
 void
 c_omp_adjust_map_clauses (tree clauses, bool is_target)
 {
@@ -3186,7 +3187,6 @@ c_omp_adjust_map_clauses (tree clauses, bool is_target)
 	    && POINTER_TYPE_P (TREE_TYPE (OMP_CLAUSE_DECL (c))))
 	  {
 	    tree ptr = OMP_CLAUSE_DECL (c);
-	    OMP_CLAUSE_SET_MAP_KIND (c, GOMP_MAP_ATTACH_DETACH);
 	    c_common_mark_addressable_vec (ptr);
 	  }
       return;
@@ -3199,7 +3199,7 @@ c_omp_adjust_map_clauses (tree clauses, bool is_target)
 	&& DECL_P (OMP_CLAUSE_DECL (c)))
       {
 	/* If this is for a target construct, the firstprivate pointer
-	   is changed to attach/detach if either is true:
+	   is marked addressable if either is true:
 	   (1) the base-pointer is mapped in this same construct, or
 	   (2) the base-pointer is a variable place on the device by
 	       "declare target" directives.
@@ -3241,11 +3241,874 @@ c_omp_adjust_map_clauses (tree clauses, bool is_target)
 
       if (mc.firstprivate_ptr_p
 	  && (mc.decl_mapped || mc.omp_declare_target))
+	c_common_mark_addressable_vec (OMP_CLAUSE_DECL (mc.clause));
+    }
+}
+
+/* Maybe strip off an indirection from a "converted" reference, then find the
+   origin of a pointer (i.e. without any offset).  */
+
+tree
+c_omp_address_inspector::unconverted_ref_origin ()
+{
+  tree t = orig;
+
+  /* We may have a reference-typed component access at the outermost level
+     that has had convert_from_reference called on it.  Get the un-dereferenced
+     reference itself.  */
+  t = maybe_unconvert_ref (t);
+
+  /* Find base pointer for POINTER_PLUS_EXPR, etc.  */
+  t = get_origin (t);
+
+  return t;
+}
+
+/* Return TRUE if the address is a component access.  */
+
+bool
+c_omp_address_inspector::component_access_p ()
+{
+  tree t = maybe_unconvert_ref (orig);
+
+  t = get_origin (t);
+
+  return TREE_CODE (t) == COMPONENT_REF;
+}
+
+/* Perform various checks on the address, as described by clause CLAUSE (we
+   only use its code and location here).  */
+
+bool
+c_omp_address_inspector::check_clause (tree clause)
+{
+  tree t = unconverted_ref_origin ();
+
+  if (TREE_CODE (t) != COMPONENT_REF)
+    return true;
+
+  if (TREE_CODE (TREE_OPERAND (t, 1)) == FIELD_DECL
+      && DECL_BIT_FIELD (TREE_OPERAND (t, 1)))
+    {
+      error_at (OMP_CLAUSE_LOCATION (clause),
+		"bit-field %qE in %qs clause",
+		t, omp_clause_code_name[OMP_CLAUSE_CODE (clause)]);
+      return false;
+    }
+  else if (!processing_template_decl_p ()
+	   && !omp_mappable_type (TREE_TYPE (t)))
+    {
+      error_at (OMP_CLAUSE_LOCATION (clause),
+		"%qE does not have a mappable type in %qs clause",
+		t, omp_clause_code_name[OMP_CLAUSE_CODE (clause)]);
+      emit_unmappable_type_notes (TREE_TYPE (t));
+      return false;
+    }
+  else if (TREE_TYPE (t) && TYPE_ATOMIC (TREE_TYPE (t)))
+    {
+      error_at (OMP_CLAUSE_LOCATION (clause),
+		"%<_Atomic%> %qE in %qs clause", t,
+		omp_clause_code_name[OMP_CLAUSE_CODE (clause)]);
+      return false;
+    }
+
+  return true;
+}
+
+/* Find the "root term" for the address.  This is the innermost decl, etc.
+   of the access.  */
+
+tree
+c_omp_address_inspector::get_root_term (bool checking)
+{
+  if (root_term && !checking)
+    return root_term;
+
+  tree t = unconverted_ref_origin ();
+
+  while (TREE_CODE (t) == COMPONENT_REF)
+    {
+      if (checking
+	  && TREE_TYPE (TREE_OPERAND (t, 0))
+	  && TREE_CODE (TREE_TYPE (TREE_OPERAND (t, 0))) == UNION_TYPE)
 	{
-	  OMP_CLAUSE_SET_MAP_KIND (mc.clause, GOMP_MAP_ATTACH_DETACH);
-	  c_common_mark_addressable_vec (OMP_CLAUSE_DECL (mc.clause));
+	  error_at (loc, "%qE is a member of a union", t);
+	  return error_mark_node;
+	}
+      t = TREE_OPERAND (t, 0);
+      while (TREE_CODE (t) == MEM_REF
+	     || TREE_CODE (t) == INDIRECT_REF
+	     || TREE_CODE (t) == ARRAY_REF)
+	{
+	  if (TREE_CODE (t) == MEM_REF
+	      || TREE_CODE (t) == INDIRECT_REF)
+	    indirections = true;
+	  t = TREE_OPERAND (t, 0);
+	  STRIP_NOPS (t);
+	  if (TREE_CODE (t) == POINTER_PLUS_EXPR)
+	    t = TREE_OPERAND (t, 0);
 	}
     }
+
+  root_term = t;
+
+  return t;
+}
+
+/* Return TRUE if the address is supported in mapping clauses.  At present,
+   this means that the innermost expression is a DECL_P, but could be extended
+   to other types of expression in the future.  */
+
+bool
+c_omp_address_inspector::map_supported_p ()
+{
+  /* If we've already decided if the mapped address is supported, return
+     that.  */
+  if (map_supported != -1)
+    return map_supported;
+
+  tree t = unconverted_ref_origin ();
+
+  STRIP_NOPS (t);
+
+  while (TREE_CODE (t) == INDIRECT_REF
+	 || TREE_CODE (t) == MEM_REF
+	 || TREE_CODE (t) == ARRAY_REF
+	 || TREE_CODE (t) == COMPONENT_REF
+	 || TREE_CODE (t) == COMPOUND_EXPR
+	 || TREE_CODE (t) == SAVE_EXPR
+	 || TREE_CODE (t) == POINTER_PLUS_EXPR
+	 || TREE_CODE (t) == NON_LVALUE_EXPR
+	 || TREE_CODE (t) == NOP_EXPR)
+    if (TREE_CODE (t) == COMPOUND_EXPR)
+      t = TREE_OPERAND (t, 1);
+    else
+      t = TREE_OPERAND (t, 0);
+
+  STRIP_NOPS (t);
+
+  map_supported = DECL_P (t);
+
+  return map_supported;
+}
+
+/* Get the origin of an address T, stripping off offsets and some other
+   bits.  */
+
+tree
+c_omp_address_inspector::get_origin (tree t)
+{
+  while (1)
+    {
+      if (TREE_CODE (t) == COMPOUND_EXPR)
+	{
+	  t = TREE_OPERAND (t, 1);
+	  STRIP_NOPS (t);
+	}
+      else if (TREE_CODE (t) == POINTER_PLUS_EXPR
+	       || TREE_CODE (t) == SAVE_EXPR)
+	t = TREE_OPERAND (t, 0);
+      else if (TREE_CODE (t) == INDIRECT_REF
+	       && TREE_CODE (TREE_TYPE (TREE_OPERAND (t, 0))) == REFERENCE_TYPE)
+	t = TREE_OPERAND (t, 0);
+      else
+	break;
+    }
+  STRIP_NOPS (t);
+  return t;
+}
+
+/* For an address T that might be a reference that has had
+   "convert_from_reference" called on it, return the actual reference without
+   any indirection.  */
+
+tree
+c_omp_address_inspector::maybe_unconvert_ref (tree t)
+{
+  if (TREE_CODE (t) == INDIRECT_REF
+      && TREE_CODE (TREE_TYPE (TREE_OPERAND (t, 0))) == REFERENCE_TYPE)
+    return TREE_OPERAND (t, 0);
+
+  return t;
+}
+
+/* Return TRUE if CLAUSE might describe a zero-length array section.  */
+
+bool
+c_omp_address_inspector::maybe_zero_length_array_section (tree clause)
+{
+  switch (OMP_CLAUSE_MAP_KIND (clause))
+    {
+    case GOMP_MAP_ALLOC:
+    case GOMP_MAP_IF_PRESENT:
+    case GOMP_MAP_TO:
+    case GOMP_MAP_FROM:
+    case GOMP_MAP_TOFROM:
+    case GOMP_MAP_ALWAYS_TO:
+    case GOMP_MAP_ALWAYS_FROM:
+    case GOMP_MAP_ALWAYS_TOFROM:
+    case GOMP_MAP_PRESENT_ALLOC:
+    case GOMP_MAP_PRESENT_TO:
+    case GOMP_MAP_PRESENT_FROM:
+    case GOMP_MAP_PRESENT_TOFROM:
+    case GOMP_MAP_ALWAYS_PRESENT_TO:
+    case GOMP_MAP_ALWAYS_PRESENT_FROM:
+    case GOMP_MAP_ALWAYS_PRESENT_TOFROM:
+    case GOMP_MAP_RELEASE:
+    case GOMP_MAP_DELETE:
+    case GOMP_MAP_FORCE_TO:
+    case GOMP_MAP_FORCE_FROM:
+    case GOMP_MAP_FORCE_TOFROM:
+    case GOMP_MAP_FORCE_PRESENT:
+      return true;
+    default:
+      return false;
+    }
+}
+
+/* Expand a chained access.  We only expect to see a quite limited range of
+   expression types here, because e.g. you can't have an array of
+   references.  */
+
+static tree
+omp_expand_access_chain (tree c, tree expr, vec<omp_addr_token *> &addr_tokens,
+			 unsigned *idx, c_omp_region_type ort)
+{
+  using namespace omp_addr_tokenizer;
+  location_t loc = OMP_CLAUSE_LOCATION (c);
+  unsigned i = *idx;
+  tree c2 = NULL_TREE;
+  gomp_map_kind kind;
+
+  if ((ort & C_ORT_EXIT_DATA) != 0
+      || OMP_CLAUSE_CODE (c) == OMP_CLAUSE_FROM
+      || (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_MAP
+	  && (OMP_CLAUSE_MAP_KIND (c) == GOMP_MAP_FROM
+	      || OMP_CLAUSE_MAP_KIND (c) == GOMP_MAP_DELETE
+	      || OMP_CLAUSE_MAP_KIND (c) == GOMP_MAP_RELEASE
+	      || OMP_CLAUSE_MAP_KIND (c) == GOMP_MAP_ALWAYS_FROM
+	      || OMP_CLAUSE_MAP_KIND (c) == GOMP_MAP_FORCE_FROM
+	      || OMP_CLAUSE_MAP_KIND (c) == GOMP_MAP_PRESENT_FROM
+	      || OMP_CLAUSE_MAP_KIND (c) == GOMP_MAP_ALWAYS_PRESENT_FROM)))
+    kind = GOMP_MAP_DETACH;
+  else
+    kind = GOMP_MAP_ATTACH;
+
+  switch (addr_tokens[i]->u.access_kind)
+    {
+    case ACCESS_POINTER:
+    case ACCESS_POINTER_OFFSET:
+      {
+	tree virtual_origin
+	  = fold_convert_loc (loc, ptrdiff_type_node, addr_tokens[i]->expr);
+	tree data_addr = omp_accessed_addr (addr_tokens, i, expr);
+	c2 = build_omp_clause (loc, OMP_CLAUSE_MAP);
+	OMP_CLAUSE_SET_MAP_KIND (c2, kind);
+	OMP_CLAUSE_DECL (c2) = addr_tokens[i]->expr;
+	OMP_CLAUSE_SIZE (c2)
+	  = fold_build2_loc (loc, MINUS_EXPR, ptrdiff_type_node,
+			     fold_convert_loc (loc, ptrdiff_type_node,
+					       data_addr),
+			     virtual_origin);
+      }
+      break;
+
+    case ACCESS_INDEXED_ARRAY:
+      break;
+
+    default:
+      return error_mark_node;
+    }
+
+  if (c2)
+    {
+      OMP_CLAUSE_CHAIN (c2) = OMP_CLAUSE_CHAIN (c);
+      OMP_CLAUSE_CHAIN (c) = c2;
+      c = c2;
+    }
+
+  *idx = ++i;
+
+  if (i < addr_tokens.length ()
+      && addr_tokens[i]->type == ACCESS_METHOD)
+    return omp_expand_access_chain (c, expr, addr_tokens, idx, ort);
+
+  return c;
+}
+
+/* Translate "array_base_decl access_method" to OMP mapping clauses.  */
+
+tree
+c_omp_address_inspector::expand_array_base (tree c,
+					    vec<omp_addr_token *> &addr_tokens,
+					    tree expr, unsigned *idx,
+					    c_omp_region_type ort)
+{
+  using namespace omp_addr_tokenizer;
+  location_t loc = OMP_CLAUSE_LOCATION (c);
+  int i = *idx;
+  tree decl = addr_tokens[i + 1]->expr;
+  bool decl_p = DECL_P (decl);
+  bool declare_target_p = (decl_p
+			   && is_global_var (decl)
+			   && lookup_attribute ("omp declare target",
+						DECL_ATTRIBUTES (decl)));
+  bool map_p = OMP_CLAUSE_CODE (c) == OMP_CLAUSE_MAP;
+  bool implicit_p = map_p && OMP_CLAUSE_MAP_IMPLICIT (c);
+  bool chain_p = omp_access_chain_p (addr_tokens, i + 1);
+  tree c2 = NULL_TREE, c3 = NULL_TREE;
+  unsigned consume_tokens = 2;
+  bool target_p = (ort & C_ORT_TARGET) != 0;
+  bool openmp_p = (ort & C_ORT_OMP) != 0;
+
+  gcc_assert (i == 0);
+
+  if (!openmp_p
+      && map_p
+      && (OMP_CLAUSE_MAP_KIND (c) == GOMP_MAP_ATTACH
+	  || OMP_CLAUSE_MAP_KIND (c) == GOMP_MAP_DETACH))
+    {
+      i += 2;
+      *idx = i;
+      return c;
+    }
+
+  switch (addr_tokens[i + 1]->u.access_kind)
+    {
+    case ACCESS_DIRECT:
+      if (decl_p && !target_p)
+	c_common_mark_addressable_vec (addr_tokens[i + 1]->expr);
+      break;
+
+    case ACCESS_REF:
+      {
+	/* Copy the referenced object.  Note that we do this even for !MAP_P
+	   clauses.  */
+	tree obj = convert_from_reference (addr_tokens[i + 1]->expr);
+	if (TREE_CODE (TREE_TYPE (obj)) == ARRAY_TYPE)
+	  /* We have a ref to array: add a [0] element as the ME expects.  */
+	  OMP_CLAUSE_DECL (c) = build_array_ref (loc, obj, integer_zero_node);
+	else
+	  OMP_CLAUSE_DECL (c) = obj;
+	OMP_CLAUSE_SIZE (c) = TYPE_SIZE_UNIT (TREE_TYPE (obj));
+
+	if (!map_p)
+	  {
+	    if (decl_p)
+	      c_common_mark_addressable_vec (addr_tokens[i + 1]->expr);
+	    break;
+	  }
+
+	if (!target_p)
+	  break;
+
+	/* If we have a reference to a pointer, avoid using
+	   FIRSTPRIVATE_REFERENCE here in case the pointer is modified in the
+	   offload region (we can only do that if the pointer does not point
+	   to a mapped block).  We could avoid doing this if we don't have a
+	   FROM mapping...  */
+	bool ref_to_ptr = TREE_CODE (TREE_TYPE (obj)) == POINTER_TYPE;
+
+	c2 = build_omp_clause (loc, OMP_CLAUSE_MAP);
+	if (!ref_to_ptr
+	    && !declare_target_p
+	    && decl_p)
+	  OMP_CLAUSE_SET_MAP_KIND (c2, GOMP_MAP_FIRSTPRIVATE_REFERENCE);
+	else
+	  {
+	    OMP_CLAUSE_SET_MAP_KIND (c2, GOMP_MAP_ATTACH_DETACH);
+	    if (decl_p)
+	      c_common_mark_addressable_vec (addr_tokens[i + 1]->expr);
+	  }
+	OMP_CLAUSE_DECL (c2) = addr_tokens[i + 1]->expr;
+	OMP_CLAUSE_SIZE (c2) = size_zero_node;
+
+	if (ref_to_ptr)
+	  {
+	    c3 = c2;
+	    c2 = build_omp_clause (loc, OMP_CLAUSE_MAP);
+	    OMP_CLAUSE_SET_MAP_KIND (c2, GOMP_MAP_ALLOC);
+	    OMP_CLAUSE_DECL (c2) = addr_tokens[i + 1]->expr;
+	    OMP_CLAUSE_SIZE (c2)
+	      = TYPE_SIZE_UNIT (TREE_TYPE (OMP_CLAUSE_DECL (c2)));
+	  }
+      }
+      break;
+
+    case ACCESS_INDEXED_REF_TO_ARRAY:
+      {
+	if (!map_p)
+	  {
+	    if (decl_p)
+	      c_common_mark_addressable_vec (addr_tokens[i + 1]->expr);
+	    break;
+	  }
+
+	if (!target_p)
+	  break;
+
+	tree virtual_origin
+	  = convert_from_reference (addr_tokens[i + 1]->expr);
+	virtual_origin = build_fold_addr_expr (virtual_origin);
+	virtual_origin = fold_convert_loc (loc, ptrdiff_type_node,
+					   virtual_origin);
+	tree data_addr = omp_accessed_addr (addr_tokens, i + 1, expr);
+	c2 = build_omp_clause (loc, OMP_CLAUSE_MAP);
+	if (decl_p && target_p && !declare_target_p)
+	  {
+	    /* It appears that omp-low.cc mishandles cases where we have a
+	       [reference to an] array of pointers such as:
+
+		 int *arr[N];   (or "int *(&arr)[N] = ...")
+		 #pragma omp target map(arr[a][b:c])
+		 { ... }
+
+	       in such cases chain_p will be true.  For now, fall back to
+	       GOMP_MAP_POINTER.  */
+	    enum gomp_map_kind k = chain_p ? GOMP_MAP_POINTER
+					   : GOMP_MAP_FIRSTPRIVATE_REFERENCE;
+	    OMP_CLAUSE_SET_MAP_KIND (c2, k);
+	  }
+	else
+	  {
+	    if (decl_p)
+	      c_common_mark_addressable_vec (addr_tokens[i + 1]->expr);
+	    OMP_CLAUSE_SET_MAP_KIND (c2, GOMP_MAP_ATTACH_DETACH);
+	  }
+	OMP_CLAUSE_DECL (c2) = addr_tokens[i + 1]->expr;
+	OMP_CLAUSE_SIZE (c2)
+	  = fold_build2_loc (loc, MINUS_EXPR, ptrdiff_type_node,
+			     fold_convert_loc (loc, ptrdiff_type_node,
+					       data_addr),
+			     virtual_origin);
+      }
+      break;
+
+    case ACCESS_INDEXED_ARRAY:
+      {
+	if (!map_p)
+	  {
+	    if (decl_p)
+	      c_common_mark_addressable_vec (addr_tokens[i + 1]->expr);
+	    break;
+	  }
+
+	/* The code handling "firstprivatize_array_bases" in gimplify.cc is
+	   relevant here.  What do we need to create for arrays at this
+	   stage?  (This condition doesn't feel quite right.  FIXME?)  */
+	if (!target_p
+	    && (TREE_CODE (TREE_TYPE (addr_tokens[i + 1]->expr))
+		== ARRAY_TYPE))
+	  break;
+
+	tree virtual_origin
+	  = build_fold_addr_expr (addr_tokens[i + 1]->expr);
+	virtual_origin = fold_convert_loc (loc, ptrdiff_type_node,
+					   virtual_origin);
+	tree data_addr = omp_accessed_addr (addr_tokens, i + 1, expr);
+	c2 = build_omp_clause (loc, OMP_CLAUSE_MAP);
+	if (decl_p && target_p)
+	  {
+	    /* See comment for ACCESS_INDEXED_REF_TO_ARRAY above.  */
+	    enum gomp_map_kind k = chain_p ? GOMP_MAP_POINTER
+					   : GOMP_MAP_FIRSTPRIVATE_POINTER;
+	    OMP_CLAUSE_SET_MAP_KIND (c2, k);
+	  }
+	else
+	  {
+	    if (decl_p)
+	      c_common_mark_addressable_vec (addr_tokens[i + 1]->expr);
+	    OMP_CLAUSE_SET_MAP_KIND (c2, GOMP_MAP_ATTACH_DETACH);
+	  }
+	OMP_CLAUSE_DECL (c2) = addr_tokens[i + 1]->expr;
+	OMP_CLAUSE_SIZE (c2)
+	  = fold_build2_loc (loc, MINUS_EXPR, ptrdiff_type_node,
+			     fold_convert_loc (loc, ptrdiff_type_node,
+					       data_addr),
+			     virtual_origin);
+      }
+      break;
+
+    case ACCESS_POINTER:
+    case ACCESS_POINTER_OFFSET:
+      {
+	if (!map_p)
+	  {
+	    if (decl_p)
+	      c_common_mark_addressable_vec (addr_tokens[i + 1]->expr);
+	    break;
+	  }
+
+	unsigned last_access = i + 1;
+	tree virtual_origin;
+
+	if (chain_p
+	    && addr_tokens[i + 2]->type == ACCESS_METHOD
+	    && addr_tokens[i + 2]->u.access_kind == ACCESS_INDEXED_ARRAY)
+	  {
+	    /* !!! This seems wrong for ACCESS_POINTER_OFFSET.  */
+	    consume_tokens = 3;
+	    chain_p = omp_access_chain_p (addr_tokens, i + 2);
+	    last_access = i + 2;
+	    virtual_origin
+	      = build_array_ref (loc, addr_tokens[last_access]->expr,
+				 integer_zero_node);
+	    virtual_origin = build_fold_addr_expr (virtual_origin);
+	    virtual_origin = fold_convert_loc (loc, ptrdiff_type_node,
+					       virtual_origin);
+	  }
+	else
+	  virtual_origin = fold_convert_loc (loc, ptrdiff_type_node,
+					     addr_tokens[last_access]->expr);
+	tree data_addr = omp_accessed_addr (addr_tokens, last_access, expr);
+	c2 = build_omp_clause (loc, OMP_CLAUSE_MAP);
+	/* For OpenACC, use FIRSTPRIVATE_POINTER for decls even on non-compute
+	   regions (e.g. "acc data" constructs).  It'll be removed anyway in
+	   gimplify.cc, but doing it this way maintains diagnostic
+	   behaviour.  */
+	if (decl_p && (target_p || !openmp_p) && !chain_p && !declare_target_p)
+	  OMP_CLAUSE_SET_MAP_KIND (c2, GOMP_MAP_FIRSTPRIVATE_POINTER);
+	else
+	  {
+	    if (decl_p)
+	      c_common_mark_addressable_vec (addr_tokens[i + 1]->expr);
+	    OMP_CLAUSE_SET_MAP_KIND (c2, GOMP_MAP_ATTACH_DETACH);
+	  }
+	OMP_CLAUSE_DECL (c2) = addr_tokens[i + 1]->expr;
+	OMP_CLAUSE_SIZE (c2)
+	  = fold_build2_loc (loc, MINUS_EXPR, ptrdiff_type_node,
+			     fold_convert_loc (loc, ptrdiff_type_node,
+					       data_addr),
+			     virtual_origin);
+      }
+      break;
+
+    case ACCESS_REF_TO_POINTER:
+    case ACCESS_REF_TO_POINTER_OFFSET:
+      {
+	if (!map_p)
+	  {
+	    if (decl_p)
+	      c_common_mark_addressable_vec (addr_tokens[i + 1]->expr);
+	    break;
+	  }
+
+	unsigned last_access = i + 1;
+	tree virtual_origin;
+
+	if (chain_p
+	    && addr_tokens[i + 2]->type == ACCESS_METHOD
+	    && addr_tokens[i + 2]->u.access_kind == ACCESS_INDEXED_ARRAY)
+	  {
+	    /* !!! This seems wrong for ACCESS_POINTER_OFFSET.  */
+	    consume_tokens = 3;
+	    chain_p = omp_access_chain_p (addr_tokens, i + 2);
+	    last_access = i + 2;
+	    virtual_origin
+	      = build_array_ref (loc, addr_tokens[last_access]->expr,
+				 integer_zero_node);
+	    virtual_origin = build_fold_addr_expr (virtual_origin);
+	    virtual_origin = fold_convert_loc (loc, ptrdiff_type_node,
+					       virtual_origin);
+	  }
+	else
+	  {
+	    virtual_origin
+	      = convert_from_reference (addr_tokens[last_access]->expr);
+	    virtual_origin = fold_convert_loc (loc, ptrdiff_type_node,
+					       virtual_origin);
+	  }
+
+	tree data_addr = omp_accessed_addr (addr_tokens, last_access, expr);
+	c2 = build_omp_clause (loc, OMP_CLAUSE_MAP);
+	if (decl_p && target_p && !chain_p && !declare_target_p)
+	  {
+	    OMP_CLAUSE_SET_MAP_KIND (c2, GOMP_MAP_FIRSTPRIVATE_REFERENCE);
+	    OMP_CLAUSE_DECL (c2) = addr_tokens[i + 1]->expr;
+	  }
+	else
+	  {
+	    if (decl_p)
+	      c_common_mark_addressable_vec (addr_tokens[i + 1]->expr);
+	    OMP_CLAUSE_SET_MAP_KIND (c2, GOMP_MAP_ATTACH_DETACH);
+	    OMP_CLAUSE_DECL (c2)
+	      = convert_from_reference (addr_tokens[i + 1]->expr);
+	  }
+	OMP_CLAUSE_SIZE (c2)
+	  = fold_build2_loc (loc, MINUS_EXPR, ptrdiff_type_node,
+			     fold_convert_loc (loc, ptrdiff_type_node,
+					       data_addr),
+			     virtual_origin);
+      }
+      break;
+
+    default:
+      *idx = i + consume_tokens;
+      return error_mark_node;
+    }
+
+  if (c3)
+    {
+      OMP_CLAUSE_CHAIN (c3) = OMP_CLAUSE_CHAIN (c);
+      OMP_CLAUSE_CHAIN (c2) = c3;
+      OMP_CLAUSE_CHAIN (c) = c2;
+      if (implicit_p)
+	{
+	  OMP_CLAUSE_MAP_IMPLICIT (c2) = 1;
+	  OMP_CLAUSE_MAP_IMPLICIT (c3) = 1;
+	}
+      c = c3;
+    }
+  else if (c2)
+    {
+      OMP_CLAUSE_CHAIN (c2) = OMP_CLAUSE_CHAIN (c);
+      OMP_CLAUSE_CHAIN (c) = c2;
+      if (implicit_p)
+	OMP_CLAUSE_MAP_IMPLICIT (c2) = 1;
+      c = c2;
+    }
+
+  i += consume_tokens;
+  *idx = i;
+
+  if (chain_p && map_p)
+    return omp_expand_access_chain (c, expr, addr_tokens, idx, ort);
+
+  return c;
+}
+
+/* Translate "component_selector access_method" to OMP mapping clauses.  */
+
+tree
+c_omp_address_inspector::expand_component_selector (tree c,
+						    vec<omp_addr_token *>
+						      &addr_tokens,
+						    tree expr, unsigned *idx,
+						    c_omp_region_type ort)
+{
+  using namespace omp_addr_tokenizer;
+  location_t loc = OMP_CLAUSE_LOCATION (c);
+  unsigned i = *idx;
+  tree c2 = NULL_TREE, c3 = NULL_TREE;
+  bool chain_p = omp_access_chain_p (addr_tokens, i + 1);
+  bool map_p = OMP_CLAUSE_CODE (c) == OMP_CLAUSE_MAP;
+
+  switch (addr_tokens[i + 1]->u.access_kind)
+    {
+    case ACCESS_DIRECT:
+    case ACCESS_INDEXED_ARRAY:
+      break;
+
+    case ACCESS_REF:
+      {
+	/* Copy the referenced object.  Note that we also do this for !MAP_P
+	   clauses.  */
+	tree obj = convert_from_reference (addr_tokens[i + 1]->expr);
+	OMP_CLAUSE_DECL (c) = obj;
+	OMP_CLAUSE_SIZE (c) = TYPE_SIZE_UNIT (TREE_TYPE (obj));
+
+	if (!map_p)
+	  break;
+
+	c2 = build_omp_clause (loc, OMP_CLAUSE_MAP);
+	OMP_CLAUSE_SET_MAP_KIND (c2, GOMP_MAP_ATTACH_DETACH);
+	OMP_CLAUSE_DECL (c2) = addr_tokens[i + 1]->expr;
+	OMP_CLAUSE_SIZE (c2) = size_zero_node;
+      }
+      break;
+
+    case ACCESS_INDEXED_REF_TO_ARRAY:
+      {
+	if (!map_p)
+	  break;
+
+	tree virtual_origin
+	  = convert_from_reference (addr_tokens[i + 1]->expr);
+	virtual_origin = build_fold_addr_expr (virtual_origin);
+	virtual_origin = fold_convert_loc (loc, ptrdiff_type_node,
+					   virtual_origin);
+	tree data_addr = omp_accessed_addr (addr_tokens, i + 1, expr);
+
+	c2 = build_omp_clause (loc, OMP_CLAUSE_MAP);
+	OMP_CLAUSE_SET_MAP_KIND (c2, GOMP_MAP_ATTACH_DETACH);
+	OMP_CLAUSE_DECL (c2) = addr_tokens[i + 1]->expr;
+	OMP_CLAUSE_SIZE (c2)
+	  = fold_build2_loc (loc, MINUS_EXPR, ptrdiff_type_node,
+			     fold_convert_loc (loc, ptrdiff_type_node,
+					       data_addr),
+			     virtual_origin);
+      }
+      break;
+
+    case ACCESS_POINTER:
+    case ACCESS_POINTER_OFFSET:
+      {
+	if (!map_p)
+	  break;
+
+	tree virtual_origin
+	  = fold_convert_loc (loc, ptrdiff_type_node,
+			      addr_tokens[i + 1]->expr);
+	tree data_addr = omp_accessed_addr (addr_tokens, i + 1, expr);
+
+	c2 = build_omp_clause (loc, OMP_CLAUSE_MAP);
+	OMP_CLAUSE_SET_MAP_KIND (c2, GOMP_MAP_ATTACH_DETACH);
+	OMP_CLAUSE_DECL (c2) = addr_tokens[i + 1]->expr;
+	OMP_CLAUSE_SIZE (c2)
+	  = fold_build2_loc (loc, MINUS_EXPR, ptrdiff_type_node,
+			     fold_convert_loc (loc, ptrdiff_type_node,
+					       data_addr),
+			     virtual_origin);
+      }
+      break;
+
+    case ACCESS_REF_TO_POINTER:
+    case ACCESS_REF_TO_POINTER_OFFSET:
+      {
+	if (!map_p)
+	  break;
+
+	tree ptr = convert_from_reference (addr_tokens[i + 1]->expr);
+	tree virtual_origin = fold_convert_loc (loc, ptrdiff_type_node,
+						ptr);
+	tree data_addr = omp_accessed_addr (addr_tokens, i + 1, expr);
+
+	/* Attach the pointer...  */
+	c2 = build_omp_clause (OMP_CLAUSE_LOCATION (c), OMP_CLAUSE_MAP);
+	OMP_CLAUSE_SET_MAP_KIND (c2, GOMP_MAP_ATTACH_DETACH);
+	OMP_CLAUSE_DECL (c2) = ptr;
+	OMP_CLAUSE_SIZE (c2)
+	  = fold_build2_loc (loc, MINUS_EXPR, ptrdiff_type_node,
+			     fold_convert_loc (loc, ptrdiff_type_node,
+					       data_addr),
+			     virtual_origin);
+
+	/* ...and also the reference.  */
+	c3 = build_omp_clause (OMP_CLAUSE_LOCATION (c), OMP_CLAUSE_MAP);
+	OMP_CLAUSE_SET_MAP_KIND (c3, GOMP_MAP_ATTACH_DETACH);
+	OMP_CLAUSE_DECL (c3) = addr_tokens[i + 1]->expr;
+	OMP_CLAUSE_SIZE (c3) = size_zero_node;
+      }
+      break;
+
+    default:
+      *idx = i + 2;
+      return error_mark_node;
+    }
+
+  if (c3)
+    {
+      OMP_CLAUSE_CHAIN (c3) = OMP_CLAUSE_CHAIN (c);
+      OMP_CLAUSE_CHAIN (c2) = c3;
+      OMP_CLAUSE_CHAIN (c) = c2;
+      c = c3;
+    }
+  else if (c2)
+    {
+      OMP_CLAUSE_CHAIN (c2) = OMP_CLAUSE_CHAIN (c);
+      OMP_CLAUSE_CHAIN (c) = c2;
+      c = c2;
+    }
+
+  i += 2;
+  *idx = i;
+
+  if (chain_p && map_p)
+    return omp_expand_access_chain (c, expr, addr_tokens, idx, ort);
+
+  return c;
+}
+
+/* Expand a map clause into a group of mapping clauses, creating nodes to
+   attach/detach pointers and so forth as necessary.  */
+
+tree
+c_omp_address_inspector::expand_map_clause (tree c, tree expr,
+					    vec<omp_addr_token *> &addr_tokens,
+					    c_omp_region_type ort)
+{
+  using namespace omp_addr_tokenizer;
+  unsigned i, length = addr_tokens.length ();
+
+  for (i = 0; i < length;)
+    {
+      int remaining = length - i;
+
+      if (remaining >= 2
+	  && addr_tokens[i]->type == ARRAY_BASE
+	  && addr_tokens[i]->u.structure_base_kind == BASE_DECL
+	  && addr_tokens[i + 1]->type == ACCESS_METHOD)
+	{
+	  c = expand_array_base (c, addr_tokens, expr, &i, ort);
+	  if (c == error_mark_node)
+	    return error_mark_node;
+	}
+      else if (remaining >= 2
+	       && addr_tokens[i]->type == ARRAY_BASE
+	       && addr_tokens[i]->u.structure_base_kind == BASE_ARBITRARY_EXPR
+	       && addr_tokens[i + 1]->type == ACCESS_METHOD)
+	{
+	  c = expand_array_base (c, addr_tokens, expr, &i, ort);
+	  if (c == error_mark_node)
+	    return error_mark_node;
+	}
+      else if (remaining >= 2
+	       && addr_tokens[i]->type == STRUCTURE_BASE
+	       && addr_tokens[i]->u.structure_base_kind == BASE_DECL
+	       && addr_tokens[i + 1]->type == ACCESS_METHOD)
+	{
+	  if (addr_tokens[i + 1]->u.access_kind == ACCESS_DIRECT)
+	    c_common_mark_addressable_vec (addr_tokens[i + 1]->expr);
+	  i += 2;
+	  while (addr_tokens[i]->type == ACCESS_METHOD)
+	    i++;
+	}
+      else if (remaining >= 2
+	       && addr_tokens[i]->type == STRUCTURE_BASE
+	       && addr_tokens[i]->u.structure_base_kind == BASE_ARBITRARY_EXPR
+	       && addr_tokens[i + 1]->type == ACCESS_METHOD)
+	{
+	  switch (addr_tokens[i + 1]->u.access_kind)
+	    {
+	    case ACCESS_DIRECT:
+	    case ACCESS_POINTER:
+	      i += 2;
+	      while (addr_tokens[i]->type == ACCESS_METHOD)
+		i++;
+	      break;
+	    default:
+	      return error_mark_node;
+	    }
+	}
+      else if (remaining >= 2
+	       && addr_tokens[i]->type == COMPONENT_SELECTOR
+	       && addr_tokens[i + 1]->type == ACCESS_METHOD)
+	{
+	  c = expand_component_selector (c, addr_tokens, expr, &i, ort);
+	  /* We used 'expr', so these must have been the last tokens.  */
+	  gcc_assert (i == length);
+	  if (c == error_mark_node)
+	    return error_mark_node;
+	}
+      else if (remaining >= 3
+	       && addr_tokens[i]->type == COMPONENT_SELECTOR
+	       && addr_tokens[i + 1]->type == STRUCTURE_BASE
+	       && (addr_tokens[i + 1]->u.structure_base_kind
+		   == BASE_COMPONENT_EXPR)
+	       && addr_tokens[i + 2]->type == ACCESS_METHOD)
+	{
+	  i += 3;
+	  while (addr_tokens[i]->type == ACCESS_METHOD)
+	    i++;
+	}
+      else
+	break;
+    }
+
+  if (i == length)
+    return c;
+
+  return error_mark_node;
 }
 
 const struct c_omp_directive c_omp_directives[] = {

gcc/c-family/c-opts.cc

@@ -532,7 +532,7 @@ c_common_handle_option (size_t scode, const char *arg, HOST_WIDE_INT value,
       break;
 
     case OPT_fdebug_cpp:
-      cpp_opts->debug = 1;
+      cpp_opts->debug = value;
       break;
 
     case OPT_ftrack_macro_expansion:

gcc/c-family/c-warn.cc

@@ -2991,14 +2991,13 @@ check_alignment_of_packed_member (tree type, tree field, bool rvalue)
   return NULL_TREE;
 }
 
-/* Return struct or union type if the right hand value, RHS:
-   1. Is a pointer value which isn't aligned to a pointer type TYPE.
-   2. Is an address which takes the unaligned address of packed member
-      of struct or union when assigning to TYPE.
+/* Return struct or union type if the right hand value, RHS,
+   is an address which takes the unaligned address of packed member
+   of struct or union when assigning to TYPE.
    Otherwise, return NULL_TREE.  */
 
 static tree
-check_address_or_pointer_of_packed_member (tree type, tree rhs)
+check_address_of_packed_member (tree type, tree rhs)
 {
   bool rvalue = true;
   bool indirect = false;
@@ -3021,57 +3020,6 @@ check_address_or_pointer_of_packed_member (tree type, tree rhs)
 
   type = TREE_TYPE (type);
 
-  if (TREE_CODE (rhs) == PARM_DECL
-      || VAR_P (rhs)
-      || TREE_CODE (rhs) == CALL_EXPR)
-    {
-      tree rhstype = TREE_TYPE (rhs);
-      if (TREE_CODE (rhs) == CALL_EXPR)
-	{
-	  rhs = CALL_EXPR_FN (rhs);	/* Pointer expression.  */
-	  if (rhs == NULL_TREE)
-	    return NULL_TREE;
-	  rhs = TREE_TYPE (rhs);	/* Pointer type.  */
-	  /* We could be called while processing a template and RHS could be
-	     a functor.  In that case it's a class, not a pointer.  */
-	  if (!rhs || !POINTER_TYPE_P (rhs))
-	    return NULL_TREE;
-	  rhs = TREE_TYPE (rhs);	/* Function type.  */
-	  rhstype = TREE_TYPE (rhs);
-	  if (!rhstype || !POINTER_TYPE_P (rhstype))
-	    return NULL_TREE;
-	  rvalue = true;
-	}
-      if (rvalue && POINTER_TYPE_P (rhstype))
-	rhstype = TREE_TYPE (rhstype);
-      while (TREE_CODE (rhstype) == ARRAY_TYPE)
-	rhstype = TREE_TYPE (rhstype);
-      if (TYPE_PACKED (rhstype))
-	{
-	  unsigned int type_align = min_align_of_type (type);
-	  unsigned int rhs_align = min_align_of_type (rhstype);
-	  if (rhs_align < type_align)
-	    {
-	      auto_diagnostic_group d;
-	      location_t location = EXPR_LOC_OR_LOC (rhs, input_location);
-	      if (warning_at (location, OPT_Waddress_of_packed_member,
-			      "converting a packed %qT pointer (alignment %d) "
-			      "to a %qT pointer (alignment %d) may result in "
-			      "an unaligned pointer value",
-			      rhstype, rhs_align, type, type_align))
-		{
-		  tree decl = TYPE_STUB_DECL (rhstype);
-		  if (decl)
-		    inform (DECL_SOURCE_LOCATION (decl), "defined here");
-		  decl = TYPE_STUB_DECL (type);
-		  if (decl)
-		    inform (DECL_SOURCE_LOCATION (decl), "defined here");
-		}
-	    }
-	}
-      return NULL_TREE;
-    }
-
   tree context = NULL_TREE;
 
   /* Check alignment of the object.  */
@@ -3094,14 +3042,12 @@ check_address_or_pointer_of_packed_member (tree type, tree rhs)
   return context;
 }
 
-/* Check and warn if the right hand value, RHS:
-   1. Is a pointer value which isn't aligned to a pointer type TYPE.
-   2. Is an address which takes the unaligned address of packed member
-      of struct or union when assigning to TYPE.
- */
+/* Check and warn if the right hand value, RHS,
+   is an address which takes the unaligned address of packed member
+   of struct or union when assigning to TYPE.  */
 
 static void
-check_and_warn_address_or_pointer_of_packed_member (tree type, tree rhs)
+check_and_warn_address_of_packed_member (tree type, tree rhs)
 {
   bool nop_p = false;
   tree orig_rhs;
@@ -3119,11 +3065,11 @@ check_and_warn_address_or_pointer_of_packed_member (tree type, tree rhs)
   if (TREE_CODE (rhs) == COND_EXPR)
     {
       /* Check the THEN path.  */
-      check_and_warn_address_or_pointer_of_packed_member
+      check_and_warn_address_of_packed_member
 	(type, TREE_OPERAND (rhs, 1));
 
       /* Check the ELSE path.  */
-      check_and_warn_address_or_pointer_of_packed_member
+      check_and_warn_address_of_packed_member
 	(type, TREE_OPERAND (rhs, 2));
     }
   else
@@ -3147,7 +3093,7 @@ check_and_warn_address_or_pointer_of_packed_member (tree type, tree rhs)
 	}
 
       tree context
-	= check_address_or_pointer_of_packed_member (type, rhs);
+	= check_address_of_packed_member (type, rhs);
       if (context)
 	{
 	  location_t loc = EXPR_LOC_OR_LOC (rhs, input_location);
@@ -3159,14 +3105,12 @@ check_and_warn_address_or_pointer_of_packed_member (tree type, tree rhs)
     }
 }
 
-/* Warn if the right hand value, RHS:
-   1. Is a pointer value which isn't aligned to a pointer type TYPE.
-   2. Is an address which takes the unaligned address of packed member
-      of struct or union when assigning to TYPE.
-*/
+/* Warn if the right hand value, RHS,
+   is an address which takes the unaligned address of packed member
+   of struct or union when assigning to TYPE.  */
 
 void
-warn_for_address_or_pointer_of_packed_member (tree type, tree rhs)
+warn_for_address_of_packed_member (tree type, tree rhs)
 {
   if (!warn_address_of_packed_member)
     return;
@@ -3175,7 +3119,7 @@ warn_for_address_or_pointer_of_packed_member (tree type, tree rhs)
   if (!POINTER_TYPE_P (type))
     return;
 
-  check_and_warn_address_or_pointer_of_packed_member (type, rhs);
+  check_and_warn_address_of_packed_member (type, rhs);
 }
 
 /* Return EXPR + 1.  Convenience helper used below.  */

gcc/c-family/c.opt

@@ -1805,6 +1805,10 @@ fdiagnostics-show-template-tree
 C++ ObjC++ Var(flag_diagnostics_show_template_tree) Init(0)
 Print hierarchical comparisons when template types are mismatched.
 
+fdiagnostics-all-candidates
+C++ ObjC++ Var(flag_diagnostics_all_candidates)
+Note all candidates during overload resolution failure.
+
 fdirectives-only
 C ObjC C++ ObjC++
 Preprocess directives only.

gcc/c/ChangeLog

@@ -1,3 +1,44 @@
+2023-12-13  Jason Merrill  <jason@redhat.com>
+
+	* c-typeck.cc (convert_for_assignment): Adjust call to
+	warn_for_address_of_packed_member.
+
+2023-12-13  Julian Brown  <julian@codesourcery.com>
+
+	* c-parser.cc (c_parser_oacc_all_clauses): Add TARGET_P parameter. Use
+	to select region type for c_finish_omp_clauses call.
+	(c_parser_oacc_loop): Update calls to c_parser_oacc_all_clauses.
+	(c_parser_oacc_compute): Likewise.
+	(c_parser_omp_target_data, c_parser_omp_target_enter_data): Support
+	ATTACH kind.
+	(c_parser_omp_target_exit_data): Support DETACH kind.
+	(check_clauses): Handle GOMP_MAP_POINTER and GOMP_MAP_ATTACH here.
+	* c-typeck.cc (handle_omp_array_sections_1,
+	handle_omp_array_sections, c_finish_omp_clauses): Use
+	c_omp_address_inspector class and OMP address tokenizer to analyze and
+	expand map clause expressions.  Fix some diagnostics.  Fix "is OpenACC"
+	condition for C_ORT_ACC_TARGET addition.
+
+2023-12-13  Julian Brown  <julian@codesourcery.com>
+
+	* c-typeck.cc (c_finish_omp_clauses): Add braces and reindent
+	OMP_CLAUSE_TO/OMP_CLAUSE_FROM/OMP_CLAUSE__CACHE_ stanza.
+
+2023-12-11  Martin Uecker  <uecker@tugraz.at>
+
+	PR c/112488
+	* c-decl.cc (add_decl_expr): Revise.
+	(finish_struct): Create DECL_EXPR.
+	* c-parser.cc (c_parser_struct_or_union_specifier): Call
+	finish_struct with expression for VLA sizes.
+	* c-tree.h (finish_struct): Add argument.
+
+2023-12-11  Tobias Burnus  <tobias@codesourcery.com>
+
+	* c-parser.cc (c_parser_omp_requires): Handle acquires/release
+	in atomic_default_mem_order clause.
+	(c_parser_omp_atomic): Update.
+
 2023-12-05  Richard Sandiford  <richard.sandiford@arm.com>
 
 	* c-decl.cc (std_attribute_table): Add extra braces to work

gcc/c/c-decl.cc

@@ -6618,12 +6618,10 @@ smallest_type_quals_location (const location_t *locations,
    the size evaluation prior to the side effects.  We therefore
    use BIND_EXPRs in TYPENAME contexts too.  */
 static void
-add_decl_expr (location_t loc, enum decl_context decl_context, tree type,
-	       tree *expr)
+add_decl_expr (location_t loc, tree type, tree *expr, bool set_name_p)
 {
   tree bind = NULL_TREE;
-  if (decl_context == TYPENAME || decl_context == PARM
-      || decl_context == FIELD)
+  if (expr)
     {
       bind = build3 (BIND_EXPR, void_type_node, NULL_TREE, NULL_TREE,
 		     NULL_TREE);
@@ -6636,7 +6634,8 @@ add_decl_expr (location_t loc, enum decl_context decl_context, tree type,
   pushdecl (decl);
   DECL_ARTIFICIAL (decl) = 1;
   add_stmt (build_stmt (DECL_SOURCE_LOCATION (decl), DECL_EXPR, decl));
-  TYPE_NAME (type) = decl;
+  if (set_name_p)
+    TYPE_NAME (type) = decl;
 
   if (bind)
     {
@@ -7635,7 +7634,12 @@ grokdeclarator (const struct c_declarator *declarator,
 	       type has a name/declaration of it's own, but special attention
 	       is required if the type is anonymous. */
 	    if (!TYPE_NAME (type) && c_type_variably_modified_p (type))
-	      add_decl_expr (loc, decl_context, type, expr);
+	      {
+		bool bind_p = decl_context == TYPENAME
+			      || decl_context == FIELD
+			      || decl_context == PARM;
+		add_decl_expr (loc, type, bind_p ? expr : NULL, true);
+	      }
 
 	    type = c_build_pointer_type (type);
 
@@ -7900,7 +7904,12 @@ grokdeclarator (const struct c_declarator *declarator,
 
 	    /* The pointed-to type may need a decl expr (see above).  */
 	    if (!TYPE_NAME (type) && c_type_variably_modified_p (type))
-	      add_decl_expr (loc, decl_context, type, expr);
+	      {
+		bool bind_p = decl_context == TYPENAME
+			      || decl_context == FIELD
+			      || decl_context == PARM;
+		add_decl_expr (loc, type, bind_p ? expr : NULL, true);
+	      }
 
 	    type = c_build_pointer_type (type);
 	    type_quals = array_ptr_quals;
@@ -9257,7 +9266,8 @@ is_flexible_array_member_p (bool is_last_field,
 
 tree
 finish_struct (location_t loc, tree t, tree fieldlist, tree attributes,
-	       class c_struct_parse_info *enclosing_struct_parse_info)
+	       class c_struct_parse_info *enclosing_struct_parse_info,
+	       tree *expr)
 {
   tree x;
   bool toplevel = file_scope == current_scope;
@@ -9595,6 +9605,13 @@ finish_struct (location_t loc, tree t, tree fieldlist, tree attributes,
 
   finish_incomplete_vars (incomplete_vars, toplevel);
 
+  /* Make sure a DECL_EXPR is created for structs with VLA members.
+     Because we do not know the context, we always pass expr
+     to force creation of a BIND_EXPR which is required in some
+     contexts.  */
+  if (c_type_variably_modified_p (t))
+    add_decl_expr (loc, t, expr, false);
+
   if (warn_cxx_compat)
     warn_cxx_compat_finish_struct (fieldlist, TREE_CODE (t), loc);
 

gcc/c/c-parser.cc

@@ -4087,7 +4087,7 @@ c_parser_struct_or_union_specifier (c_parser *parser)
       ret.spec = finish_struct (struct_loc, type, nreverse (contents),
 				chainon (std_attrs,
 					 chainon (attrs, postfix_attrs)),
-				struct_info);
+				struct_info, &expr);
       ret.kind = ctsk_tagdef;
       ret.expr = expr;
       ret.expr_const_operands = true;
@@ -19063,7 +19063,8 @@ c_parser_omp_clause_detach (c_parser *parser, tree list)
 
 static tree
 c_parser_oacc_all_clauses (c_parser *parser, omp_clause_mask mask,
-			   const char *where, bool finish_p = true)
+			   const char *where, bool finish_p = true,
+			   bool target_p = false)
 {
   tree clauses = NULL;
   bool first = true;
@@ -19273,7 +19274,8 @@ c_parser_oacc_all_clauses (c_parser *parser, omp_clause_mask mask,
   c_parser_skip_to_pragma_eol (parser);
 
   if (finish_p)
-    return c_finish_omp_clauses (clauses, C_ORT_ACC);
+    return c_finish_omp_clauses (clauses, target_p ? C_ORT_ACC_TARGET
+						   : C_ORT_ACC);
 
   return clauses;
 }
@@ -20011,12 +20013,13 @@ c_parser_oacc_loop (location_t loc, c_parser *parser, char *p_name,
   mask |= OACC_LOOP_CLAUSE_MASK;
 
   tree clauses = c_parser_oacc_all_clauses (parser, mask, p_name,
-					    cclauses == NULL);
+					    /*finish_p=*/cclauses == NULL,
+					    /*target=*/is_parallel);
   if (cclauses)
     {
       clauses = c_oacc_split_loop_clauses (clauses, cclauses, is_parallel);
       if (*cclauses)
-	*cclauses = c_finish_omp_clauses (*cclauses, C_ORT_ACC);
+	*cclauses = c_finish_omp_clauses (*cclauses, C_ORT_ACC_TARGET);
       if (clauses)
 	clauses = c_finish_omp_clauses (clauses, C_ORT_ACC);
     }
@@ -20144,7 +20147,9 @@ c_parser_oacc_compute (location_t loc, c_parser *parser,
 	}
     }
 
-  tree clauses = c_parser_oacc_all_clauses (parser, mask, p_name);
+  tree clauses = c_parser_oacc_all_clauses (parser, mask, p_name,
+					    /*finish_p=*/true,
+					    /*target=*/true);
 
   tree block = c_begin_omp_parallel ();
   add_stmt (c_parser_omp_structured_block (parser, if_p));
@@ -20896,6 +20901,28 @@ c_parser_omp_atomic (location_t loc, c_parser *parser, bool openacc)
 	case OMP_MEMORY_ORDER_SEQ_CST:
 	  memory_order = OMP_MEMORY_ORDER_SEQ_CST;
 	  break;
+	case OMP_MEMORY_ORDER_ACQUIRE:
+	  if (code == NOP_EXPR)  /* atomic write */
+	    {
+	      error_at (loc, "%<#pragma omp atomic write%> incompatible with "
+			     "%<acquire%> clause implicitly provided by a "
+			     "%<requires%> directive");
+	      memory_order = OMP_MEMORY_ORDER_SEQ_CST;
+	    }
+	  else
+	    memory_order = OMP_MEMORY_ORDER_ACQUIRE;
+	  break;
+	case OMP_MEMORY_ORDER_RELEASE:
+	  if (code == OMP_ATOMIC_READ)
+	    {
+	      error_at (loc, "%<#pragma omp atomic read%> incompatible with "
+			     "%<release%> clause implicitly provided by a "
+			     "%<requires%> directive");
+	      memory_order = OMP_MEMORY_ORDER_SEQ_CST;
+	    }
+	  else
+	    memory_order = OMP_MEMORY_ORDER_RELEASE;
+	  break;
 	case OMP_MEMORY_ORDER_ACQ_REL:
 	  switch (code)
 	    {
@@ -23648,6 +23675,7 @@ c_parser_omp_target_data (location_t loc, c_parser *parser, bool *if_p)
 	  case GOMP_MAP_FIRSTPRIVATE_POINTER:
 	  case GOMP_MAP_ALWAYS_POINTER:
 	  case GOMP_MAP_ATTACH_DETACH:
+	  case GOMP_MAP_ATTACH:
 	    break;
 	  default:
 	    map_seen |= 1;
@@ -23813,6 +23841,7 @@ c_parser_omp_target_enter_data (location_t loc, c_parser *parser,
 	  case GOMP_MAP_FIRSTPRIVATE_POINTER:
 	  case GOMP_MAP_ALWAYS_POINTER:
 	  case GOMP_MAP_ATTACH_DETACH:
+	  case GOMP_MAP_ATTACH:
 	    break;
 	  default:
 	    map_seen |= 1;
@@ -23887,7 +23916,8 @@ c_parser_omp_target_exit_data (location_t loc, c_parser *parser,
 
   tree clauses
     = c_parser_omp_all_clauses (parser, OMP_TARGET_EXIT_DATA_CLAUSE_MASK,
-				"#pragma omp target exit data");
+				"#pragma omp target exit data", false);
+  clauses = c_finish_omp_clauses (clauses, C_ORT_OMP_EXIT_DATA);
   c_omp_adjust_map_clauses (clauses, false);
   int map_seen = 0;
   for (tree *pc = &clauses; *pc;)
@@ -23922,6 +23952,7 @@ c_parser_omp_target_exit_data (location_t loc, c_parser *parser,
 	  case GOMP_MAP_FIRSTPRIVATE_POINTER:
 	  case GOMP_MAP_ALWAYS_POINTER:
 	  case GOMP_MAP_ATTACH_DETACH:
+	  case GOMP_MAP_DETACH:
 	    break;
 	  default:
 	    map_seen |= 1;
@@ -24178,7 +24209,9 @@ check_clauses:
 	  case GOMP_MAP_PRESENT_ALLOC:
 	  case GOMP_MAP_FIRSTPRIVATE_POINTER:
 	  case GOMP_MAP_ALWAYS_POINTER:
+	  case GOMP_MAP_POINTER:
 	  case GOMP_MAP_ATTACH_DETACH:
+	  case GOMP_MAP_ATTACH:
 	    break;
 	  default:
 	    error_at (OMP_CLAUSE_LOCATION (*pc),
@@ -25724,15 +25757,21 @@ c_parser_omp_requires (c_parser *parser)
 		      else if (!strcmp (p, "relaxed"))
 			this_req
 			  = (enum omp_requires) OMP_MEMORY_ORDER_RELAXED;
+		      else if (!strcmp (p, "release"))
+			this_req
+			  = (enum omp_requires) OMP_MEMORY_ORDER_RELEASE;
 		      else if (!strcmp (p, "acq_rel"))
 			this_req
 			  = (enum omp_requires) OMP_MEMORY_ORDER_ACQ_REL;
+		      else if (!strcmp (p, "acquire"))
+			this_req
+			  = (enum omp_requires) OMP_MEMORY_ORDER_ACQUIRE;
 		    }
 		  if (this_req == 0)
 		    {
 		      error_at (c_parser_peek_token (parser)->location,
-				"expected %<seq_cst%>, %<relaxed%> or "
-				"%<acq_rel%>");
+				"expected %<acq_rel%>, %<acquire%>, "
+				"%<relaxed%>, %<release%> or %<seq_cst%>");
 		      switch (c_parser_peek_token (parser)->type)
 			{
 			case CPP_EOF:

gcc/c/c-tree.h

@@ -656,7 +656,8 @@ extern void finish_decl (tree, location_t, tree, tree, tree);
 extern tree finish_enum (tree, tree, tree);
 extern void finish_function (location_t = input_location);
 extern tree finish_struct (location_t, tree, tree, tree,
-			   class c_struct_parse_info *);
+			   class c_struct_parse_info *,
+			   tree *expr = NULL);
 extern tree c_simulate_enum_decl (location_t, const char *,
 				  vec<string_int_pair> *);
 extern tree c_simulate_record_decl (location_t, const char *,

gcc/c/c-typeck.cc

@@ -7000,7 +7000,7 @@ convert_for_assignment (location_t location, location_t expr_loc, tree type,
 
   if (TYPE_MAIN_VARIANT (type) == TYPE_MAIN_VARIANT (rhstype))
     {
-      warn_for_address_or_pointer_of_packed_member (type, orig_rhs);
+      warn_for_address_of_packed_member (type, orig_rhs);
       return rhs;
     }
 
@@ -7658,7 +7658,7 @@ convert_for_assignment (location_t location, location_t expr_loc, tree type,
 
       /* If RHS isn't an address, check pointer or array of packed
 	 struct or union.  */
-      warn_for_address_or_pointer_of_packed_member (type, orig_rhs);
+      warn_for_address_of_packed_member (type, orig_rhs);
 
       return convert (type, rhs);
     }
@@ -13606,10 +13606,12 @@ handle_omp_array_sections_1 (tree c, tree t, vec<tree> &types,
 			     enum c_omp_region_type ort)
 {
   tree ret, low_bound, length, type;
+  bool openacc = (ort & C_ORT_ACC) != 0;
   if (TREE_CODE (t) != TREE_LIST)
     {
       if (error_operand_p (t))
 	return error_mark_node;
+      c_omp_address_inspector ai (OMP_CLAUSE_LOCATION (c), t);
       ret = t;
       if (OMP_CLAUSE_CODE (c) != OMP_CLAUSE_AFFINITY
 	  && OMP_CLAUSE_CODE (c) != OMP_CLAUSE_DEPEND
@@ -13619,59 +13621,17 @@ handle_omp_array_sections_1 (tree c, tree t, vec<tree> &types,
 		    t, omp_clause_code_name[OMP_CLAUSE_CODE (c)]);
 	  return error_mark_node;
 	}
-      while (INDIRECT_REF_P (t))
-	{
-	  t = TREE_OPERAND (t, 0);
-	  STRIP_NOPS (t);
-	  if (TREE_CODE (t) == POINTER_PLUS_EXPR)
-	    t = TREE_OPERAND (t, 0);
-	}
-      while (TREE_CODE (t) == COMPOUND_EXPR)
-	{
-	  t = TREE_OPERAND (t, 1);
-	  STRIP_NOPS (t);
-	}
-      if (TREE_CODE (t) == COMPONENT_REF
-	  && (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_MAP
-	      || OMP_CLAUSE_CODE (c) == OMP_CLAUSE_TO
-	      || OMP_CLAUSE_CODE (c) == OMP_CLAUSE_FROM))
-	{
-	  if (DECL_BIT_FIELD (TREE_OPERAND (t, 1)))
-	    {
-	      error_at (OMP_CLAUSE_LOCATION (c),
-			"bit-field %qE in %qs clause",
-			t, omp_clause_code_name[OMP_CLAUSE_CODE (c)]);
-	      return error_mark_node;
-	    }
-	  while (TREE_CODE (t) == COMPONENT_REF)
-	    {
-	      if (TREE_CODE (TREE_TYPE (TREE_OPERAND (t, 0))) == UNION_TYPE)
-		{
-		  error_at (OMP_CLAUSE_LOCATION (c),
-			    "%qE is a member of a union", t);
-		  return error_mark_node;
-		}
-	      t = TREE_OPERAND (t, 0);
-	      while (TREE_CODE (t) == MEM_REF
-		     || INDIRECT_REF_P (t)
-		     || TREE_CODE (t) == ARRAY_REF)
-		{
-		  t = TREE_OPERAND (t, 0);
-		  STRIP_NOPS (t);
-		  if (TREE_CODE (t) == POINTER_PLUS_EXPR)
-		    t = TREE_OPERAND (t, 0);
-		}
-	      if (ort == C_ORT_ACC && TREE_CODE (t) == MEM_REF)
-		{
-		  if (maybe_ne (mem_ref_offset (t), 0))
-		    error_at (OMP_CLAUSE_LOCATION (c),
-			      "cannot dereference %qE in %qs clause", t,
-			      omp_clause_code_name[OMP_CLAUSE_CODE (c)]);
-		  else
-		    t = TREE_OPERAND (t, 0);
-		}
-	    }
-	}
+      if (!ai.check_clause (c))
+	return error_mark_node;
+      else if (ai.component_access_p ()
+	       && (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_MAP
+		   || OMP_CLAUSE_CODE (c) == OMP_CLAUSE_TO
+		   || OMP_CLAUSE_CODE (c) == OMP_CLAUSE_FROM))
+	t = ai.get_root_term (true);
+      else
+	t = ai.unconverted_ref_origin ();
+      if (t == error_mark_node)
+	return error_mark_node;
       if (!VAR_P (t) && TREE_CODE (t) != PARM_DECL)
 	{
 	  if (DECL_P (t))
@@ -13766,7 +13726,7 @@ handle_omp_array_sections_1 (tree c, tree t, vec<tree> &types,
 	{
 	  error_at (OMP_CLAUSE_LOCATION (c),
 		    "expected single pointer in %qs clause",
-		    user_omp_clause_code_name (c, ort == C_ORT_ACC));
+		    user_omp_clause_code_name (c, openacc));
 	  return error_mark_node;
 	}
     }
@@ -13991,7 +13951,7 @@ handle_omp_array_sections_1 (tree c, tree t, vec<tree> &types,
 /* Handle array sections for clause C.  */
 
 static bool
-handle_omp_array_sections (tree c, enum c_omp_region_type ort)
+handle_omp_array_sections (tree &c, enum c_omp_region_type ort)
 {
   bool maybe_zero_len = false;
   unsigned int first_non_one = 0;
@@ -14200,58 +14160,47 @@ handle_omp_array_sections (tree c, enum c_omp_region_type ort)
       OMP_CLAUSE_DECL (c) = first;
       if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_HAS_DEVICE_ADDR)
 	return false;
-      if (size)
-	size = c_fully_fold (size, false, NULL);
-      OMP_CLAUSE_SIZE (c) = size;
+      /* Don't set OMP_CLAUSE_SIZE for bare attach/detach clauses.  */
       if (OMP_CLAUSE_CODE (c) != OMP_CLAUSE_MAP
-	  || (TREE_CODE (t) == COMPONENT_REF
-	      && TREE_CODE (TREE_TYPE (t)) == ARRAY_TYPE))
-	return false;
-      gcc_assert (OMP_CLAUSE_MAP_KIND (c) != GOMP_MAP_FORCE_DEVICEPTR);
-      switch (OMP_CLAUSE_MAP_KIND (c))
+	  || (OMP_CLAUSE_MAP_KIND (c) != GOMP_MAP_ATTACH
+	      && OMP_CLAUSE_MAP_KIND (c) != GOMP_MAP_DETACH
+	      && OMP_CLAUSE_MAP_KIND (c) != GOMP_MAP_FORCE_DETACH))
 	{
-	case GOMP_MAP_ALLOC:
-	case GOMP_MAP_IF_PRESENT:
-	case GOMP_MAP_TO:
-	case GOMP_MAP_FROM:
-	case GOMP_MAP_TOFROM:
-	case GOMP_MAP_ALWAYS_TO:
-	case GOMP_MAP_ALWAYS_FROM:
-	case GOMP_MAP_ALWAYS_TOFROM:
-	case GOMP_MAP_RELEASE:
-	case GOMP_MAP_DELETE:
-	case GOMP_MAP_FORCE_TO:
-	case GOMP_MAP_FORCE_FROM:
-	case GOMP_MAP_FORCE_TOFROM:
-	case GOMP_MAP_FORCE_PRESENT:
-	  OMP_CLAUSE_MAP_MAYBE_ZERO_LENGTH_ARRAY_SECTION (c) = 1;
-	  break;
-	default:
-	  break;
+	  if (size)
+	    size = c_fully_fold (size, false, NULL);
+	  OMP_CLAUSE_SIZE (c) = size;
 	}
-      tree c2 = build_omp_clause (OMP_CLAUSE_LOCATION (c), OMP_CLAUSE_MAP);
-      if (TREE_CODE (t) == COMPONENT_REF)
-	OMP_CLAUSE_SET_MAP_KIND (c2, GOMP_MAP_ATTACH_DETACH);
-      else
-	OMP_CLAUSE_SET_MAP_KIND (c2, GOMP_MAP_FIRSTPRIVATE_POINTER);
-      OMP_CLAUSE_MAP_IMPLICIT (c2) = OMP_CLAUSE_MAP_IMPLICIT (c);
-      if (OMP_CLAUSE_MAP_KIND (c2) != GOMP_MAP_FIRSTPRIVATE_POINTER
-	  && !c_mark_addressable (t))
+
+      if (OMP_CLAUSE_CODE (c) != OMP_CLAUSE_MAP)
 	return false;
-      OMP_CLAUSE_DECL (c2) = t;
-      t = build_fold_addr_expr (first);
-      t = fold_convert_loc (OMP_CLAUSE_LOCATION (c), ptrdiff_type_node, t);
-      tree ptr = OMP_CLAUSE_DECL (c2);
-      if (!POINTER_TYPE_P (TREE_TYPE (ptr)))
-	ptr = build_fold_addr_expr (ptr);
-      t = fold_build2_loc (OMP_CLAUSE_LOCATION (c), MINUS_EXPR,
-			   ptrdiff_type_node, t,
-			   fold_convert_loc (OMP_CLAUSE_LOCATION (c),
-					     ptrdiff_type_node, ptr));
-      t = c_fully_fold (t, false, NULL);
-      OMP_CLAUSE_SIZE (c2) = t;
-      OMP_CLAUSE_CHAIN (c2) = OMP_CLAUSE_CHAIN (c);
-      OMP_CLAUSE_CHAIN (c) = c2;
+
+      auto_vec<omp_addr_token *, 10> addr_tokens;
+
+      if (!omp_parse_expr (addr_tokens, first))
+	return true;
+
+      c_omp_address_inspector ai (OMP_CLAUSE_LOCATION (c), t);
+
+      tree nc = ai.expand_map_clause (c, first, addr_tokens, ort);
+      if (nc != error_mark_node)
+	{
+	  using namespace omp_addr_tokenizer;
+
+	  if (ai.maybe_zero_length_array_section (c))
+	    OMP_CLAUSE_MAP_MAYBE_ZERO_LENGTH_ARRAY_SECTION (c) = 1;
+
+	  /* !!! If we're accessing a base decl via chained access
+	     methods (e.g. multiple indirections), duplicate clause
+	     detection won't work properly.  Skip it in that case.  */
+	  if ((addr_tokens[0]->type == STRUCTURE_BASE
+	       || addr_tokens[0]->type == ARRAY_BASE)
+	      && addr_tokens[0]->u.structure_base_kind == BASE_DECL
+	      && addr_tokens[1]->type == ACCESS_METHOD
+	      && omp_access_chain_p (addr_tokens, 1))
+	    c = nc;
+
+	  return false;
+	}
     }
   return false;
 }
@@ -14517,7 +14466,6 @@ c_finish_omp_clauses (tree clauses, enum c_omp_region_type ort)
   tree ordered_clause = NULL_TREE;
   tree schedule_clause = NULL_TREE;
   bool oacc_async = false;
-  bool indir_component_ref_p = false;
   tree last_iterators = NULL_TREE;
   bool last_iterators_remove = false;
   tree *nogroup_seen = NULL;
@@ -14528,6 +14476,7 @@ c_finish_omp_clauses (tree clauses, enum c_omp_region_type ort)
   bool allocate_seen = false;
   bool implicit_moved = false;
   bool target_in_reduction_seen = false;
+  bool openacc = (ort & C_ORT_ACC) != 0;
 
   bitmap_obstack_initialize (NULL);
   bitmap_initialize (&generic_head, &bitmap_default_obstack);
@@ -14543,7 +14492,7 @@ c_finish_omp_clauses (tree clauses, enum c_omp_region_type ort)
   bitmap_initialize (&oacc_reduction_head, &bitmap_default_obstack);
   bitmap_initialize (&is_on_device_head, &bitmap_default_obstack);
 
-  if (ort & C_ORT_ACC)
+  if (openacc)
     for (c = clauses; c; c = OMP_CLAUSE_CHAIN (c))
       if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_ASYNC)
 	{
@@ -14937,8 +14886,7 @@ c_finish_omp_clauses (tree clauses, enum c_omp_region_type ort)
 			omp_clause_code_name[OMP_CLAUSE_CODE (c)]);
 	      remove = true;
 	    }
-	  else if ((ort == C_ORT_ACC
-		    && OMP_CLAUSE_CODE (c) == OMP_CLAUSE_REDUCTION)
+	  else if ((openacc && OMP_CLAUSE_CODE (c) == OMP_CLAUSE_REDUCTION)
 		   || (ort == C_ORT_OMP
 		       && (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_USE_DEVICE_PTR
 			   || (OMP_CLAUSE_CODE (c)
@@ -14961,7 +14909,7 @@ c_finish_omp_clauses (tree clauses, enum c_omp_region_type ort)
 	      if (bitmap_bit_p (&oacc_reduction_head, DECL_UID (t)))
 		{
 		  error_at (OMP_CLAUSE_LOCATION (c),
-			    ort == C_ORT_ACC
+			    openacc
 			    ? "%qD appears more than once in reduction clauses"
 			    : "%qD appears more than once in data clauses",
 			    t);
@@ -14984,7 +14932,7 @@ c_finish_omp_clauses (tree clauses, enum c_omp_region_type ort)
 		    || OMP_CLAUSE_CODE (c) == OMP_CLAUSE_IS_DEVICE_PTR)
 		   && bitmap_bit_p (&map_head, DECL_UID (t)))
 	    {
-	      if (ort == C_ORT_ACC)
+	      if (openacc)
 		error_at (OMP_CLAUSE_LOCATION (c),
 			  "%qD appears more than once in data clauses", t);
 	      else
@@ -15049,9 +14997,10 @@ c_finish_omp_clauses (tree clauses, enum c_omp_region_type ort)
 			"%qE appears more than once in data clauses", t);
 	      remove = true;
 	    }
-	  else if (bitmap_bit_p (&map_head, DECL_UID (t)))
+	  else if (bitmap_bit_p (&map_head, DECL_UID (t))
+		   || bitmap_bit_p (&map_field_head, DECL_UID (t)))
 	    {
-	      if (ort == C_ORT_ACC)
+	      if (openacc)
 		error_at (OMP_CLAUSE_LOCATION (c),
 			  "%qD appears more than once in data clauses", t);
 	      else if (OMP_CLAUSE_FIRSTPRIVATE_IMPLICIT (c)
@@ -15318,321 +15267,316 @@ c_finish_omp_clauses (tree clauses, enum c_omp_region_type ort)
 	case OMP_CLAUSE_TO:
 	case OMP_CLAUSE_FROM:
 	case OMP_CLAUSE__CACHE_:
-	  t = OMP_CLAUSE_DECL (c);
-	  if (TREE_CODE (t) == TREE_LIST)
-	    {
-	      grp_start_p = pc;
-	      grp_sentinel = OMP_CLAUSE_CHAIN (c);
+	  {
+	    using namespace omp_addr_tokenizer;
+	    auto_vec<omp_addr_token *, 10> addr_tokens;
 
-	      if (handle_omp_array_sections (c, ort))
-		remove = true;
-	      else
-		{
-		  t = OMP_CLAUSE_DECL (c);
-		  if (!omp_mappable_type (TREE_TYPE (t)))
-		    {
-		      error_at (OMP_CLAUSE_LOCATION (c),
-				"array section does not have mappable type "
-				"in %qs clause",
-				omp_clause_code_name[OMP_CLAUSE_CODE (c)]);
-		      remove = true;
-		    }
-		  else if (TYPE_ATOMIC (TREE_TYPE (t)))
-		    {
-		      error_at (OMP_CLAUSE_LOCATION (c),
-				"%<_Atomic%> %qE in %qs clause", t,
-				omp_clause_code_name[OMP_CLAUSE_CODE (c)]);
-		      remove = true;
-		    }
-		  while (TREE_CODE (t) == ARRAY_REF)
-		    t = TREE_OPERAND (t, 0);
-		  if (TREE_CODE (t) == COMPONENT_REF
-		      && TREE_CODE (TREE_TYPE (t)) == ARRAY_TYPE)
-		    {
-		      do
-			{
-			  t = TREE_OPERAND (t, 0);
-			  if (TREE_CODE (t) == MEM_REF
-			      || INDIRECT_REF_P (t))
-			    {
-			      t = TREE_OPERAND (t, 0);
-			      STRIP_NOPS (t);
-			      if (TREE_CODE (t) == POINTER_PLUS_EXPR)
-				t = TREE_OPERAND (t, 0);
-			    }
-			}
-		      while (TREE_CODE (t) == COMPONENT_REF
-			     || TREE_CODE (t) == ARRAY_REF);
+	    t = OMP_CLAUSE_DECL (c);
+	    if (TREE_CODE (t) == TREE_LIST)
+	      {
+		grp_start_p = pc;
+		grp_sentinel = OMP_CLAUSE_CHAIN (c);
 
-		      if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_MAP
-			  && OMP_CLAUSE_MAP_IMPLICIT (c)
-			  && (bitmap_bit_p (&map_head, DECL_UID (t))
-			      || bitmap_bit_p (&map_field_head, DECL_UID (t))
-			      || bitmap_bit_p (&map_firstprivate_head,
-					       DECL_UID (t))))
-			{
-			  remove = true;
-			  break;
-			}
-		      if (bitmap_bit_p (&map_field_head, DECL_UID (t)))
-			break;
-		      if (bitmap_bit_p (&map_head, DECL_UID (t)))
-			{
-			  if (OMP_CLAUSE_CODE (c) != OMP_CLAUSE_MAP)
-			    error_at (OMP_CLAUSE_LOCATION (c),
-				      "%qD appears more than once in motion "
-				      "clauses", t);
-			  else if (ort == C_ORT_ACC)
-			    error_at (OMP_CLAUSE_LOCATION (c),
-				      "%qD appears more than once in data "
-				      "clauses", t);
-			  else
-			    error_at (OMP_CLAUSE_LOCATION (c),
-				      "%qD appears more than once in map "
-				      "clauses", t);
-			  remove = true;
-			}
-		      else
-			{
-			  bitmap_set_bit (&map_head, DECL_UID (t));
-			  bitmap_set_bit (&map_field_head, DECL_UID (t));
-			}
-		    }
-		}
-	      if (c_oacc_check_attachments (c))
-		remove = true;
-	      if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_MAP
-		  && (OMP_CLAUSE_MAP_KIND (c) == GOMP_MAP_ATTACH
-		      || OMP_CLAUSE_MAP_KIND (c) == GOMP_MAP_DETACH))
-		/* In this case, we have a single array element which is a
-		   pointer, and we already set OMP_CLAUSE_SIZE in
-		   handle_omp_array_sections above.  For attach/detach clauses,
-		   reset the OMP_CLAUSE_SIZE (representing a bias) to zero
-		   here.  */
-		OMP_CLAUSE_SIZE (c) = size_zero_node;
-	      break;
-	    }
-	  if (t == error_mark_node)
-	    {
-	      remove = true;
-	      break;
-	    }
-	  /* OpenACC attach / detach clauses must be pointers.  */
-	  if (c_oacc_check_attachments (c))
-	    {
-	      remove = true;
-	      break;
-	    }
-	  if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_MAP
-	      && (OMP_CLAUSE_MAP_KIND (c) == GOMP_MAP_ATTACH
-		  || OMP_CLAUSE_MAP_KIND (c) == GOMP_MAP_DETACH))
-	    /* For attach/detach clauses, set OMP_CLAUSE_SIZE (representing a
-	       bias) to zero here, so it is not set erroneously to the pointer
-	       size later on in gimplify.cc.  */
-	    OMP_CLAUSE_SIZE (c) = size_zero_node;
-	  while (INDIRECT_REF_P (t)
-		 || TREE_CODE (t) == ARRAY_REF)
-	    {
-	      t = TREE_OPERAND (t, 0);
-	      STRIP_NOPS (t);
-	      if (TREE_CODE (t) == POINTER_PLUS_EXPR)
-		t = TREE_OPERAND (t, 0);
-	    }
-	  while (TREE_CODE (t) == COMPOUND_EXPR)
-	    {
-	      t = TREE_OPERAND (t, 1);
-	      STRIP_NOPS (t);
-	    }
-	  indir_component_ref_p = false;
-	  if (TREE_CODE (t) == COMPONENT_REF
-	      && (TREE_CODE (TREE_OPERAND (t, 0)) == MEM_REF
-		  || INDIRECT_REF_P (TREE_OPERAND (t, 0))
-		  || TREE_CODE (TREE_OPERAND (t, 0)) == ARRAY_REF))
-	    {
-	      t = TREE_OPERAND (TREE_OPERAND (t, 0), 0);
-	      indir_component_ref_p = true;
-	      STRIP_NOPS (t);
-	      if (TREE_CODE (t) == POINTER_PLUS_EXPR)
-		t = TREE_OPERAND (t, 0);
-	    }
-
-	  if (TREE_CODE (t) == COMPONENT_REF
-	      && OMP_CLAUSE_CODE (c) != OMP_CLAUSE__CACHE_)
-	    {
-	      if (DECL_BIT_FIELD (TREE_OPERAND (t, 1)))
-		{
-		  error_at (OMP_CLAUSE_LOCATION (c),
-			    "bit-field %qE in %qs clause",
-			    t, omp_clause_code_name[OMP_CLAUSE_CODE (c)]);
+		if (handle_omp_array_sections (c, ort))
 		  remove = true;
-		}
-	      else if (!omp_mappable_type (TREE_TYPE (t)))
-		{
-		  error_at (OMP_CLAUSE_LOCATION (c),
-			    "%qE does not have a mappable type in %qs clause",
-			    t, omp_clause_code_name[OMP_CLAUSE_CODE (c)]);
-		  remove = true;
-		}
-	      else if (TYPE_ATOMIC (TREE_TYPE (t)))
-		{
-		  error_at (OMP_CLAUSE_LOCATION (c),
-			    "%<_Atomic%> %qE in %qs clause", t,
-			    omp_clause_code_name[OMP_CLAUSE_CODE (c)]);
-		  remove = true;
-		}
-	      while (TREE_CODE (t) == COMPONENT_REF)
-		{
-		  if (TREE_CODE (TREE_TYPE (TREE_OPERAND (t, 0)))
-		      == UNION_TYPE)
-		    {
-		      error_at (OMP_CLAUSE_LOCATION (c),
-				"%qE is a member of a union", t);
-		      remove = true;
-		      break;
-		    }
-		  t = TREE_OPERAND (t, 0);
-		  if (TREE_CODE (t) == MEM_REF)
-		    {
-		      if (maybe_ne (mem_ref_offset (t), 0))
+		else
+		  {
+		    t = OMP_CLAUSE_DECL (c);
+		    if (!omp_mappable_type (TREE_TYPE (t)))
+		      {
 			error_at (OMP_CLAUSE_LOCATION (c),
-				  "cannot dereference %qE in %qs clause", t,
+				  "array section does not have mappable type "
+				  "in %qs clause",
 				  omp_clause_code_name[OMP_CLAUSE_CODE (c)]);
-		      else
-			t = TREE_OPERAND (t, 0);
-		    }
-		  while (TREE_CODE (t) == MEM_REF
-			 || INDIRECT_REF_P (t)
-			 || TREE_CODE (t) == ARRAY_REF)
-		    {
+			remove = true;
+		      }
+		    else if (TYPE_ATOMIC (TREE_TYPE (t)))
+		      {
+			error_at (OMP_CLAUSE_LOCATION (c),
+				  "%<_Atomic%> %qE in %qs clause", t,
+				  omp_clause_code_name[OMP_CLAUSE_CODE (c)]);
+			remove = true;
+		      }
+		    while (TREE_CODE (t) == ARRAY_REF)
 		      t = TREE_OPERAND (t, 0);
-		      STRIP_NOPS (t);
-		      if (TREE_CODE (t) == POINTER_PLUS_EXPR)
-			t = TREE_OPERAND (t, 0);
-		    }
-		}
-	      if (remove)
-		break;
-	      if (VAR_P (t) || TREE_CODE (t) == PARM_DECL)
-		{
-		  if (bitmap_bit_p (&map_field_head, DECL_UID (t))
-		      || (ort != C_ORT_ACC
-			  && bitmap_bit_p (&map_head, DECL_UID (t))))
-		    break;
-		}
-	    }
-	  if (!VAR_P (t) && TREE_CODE (t) != PARM_DECL)
-	    {
-	      error_at (OMP_CLAUSE_LOCATION (c),
-			"%qE is not a variable in %qs clause", t,
-			omp_clause_code_name[OMP_CLAUSE_CODE (c)]);
-	      remove = true;
-	    }
-	  else if (VAR_P (t) && DECL_THREAD_LOCAL_P (t))
-	    {
-	      error_at (OMP_CLAUSE_LOCATION (c),
-			"%qD is threadprivate variable in %qs clause", t,
-			omp_clause_code_name[OMP_CLAUSE_CODE (c)]);
-	      remove = true;
-	    }
-	  else if ((OMP_CLAUSE_CODE (c) != OMP_CLAUSE_MAP
-		    || (OMP_CLAUSE_MAP_KIND (c)
-			!= GOMP_MAP_FIRSTPRIVATE_POINTER))
-		   && !indir_component_ref_p
-		   && !c_mark_addressable (t))
-	    remove = true;
-	  else if (!(OMP_CLAUSE_CODE (c) == OMP_CLAUSE_MAP
-		     && (OMP_CLAUSE_MAP_KIND (c) == GOMP_MAP_POINTER
-			 || (OMP_CLAUSE_MAP_KIND (c)
-			     == GOMP_MAP_FIRSTPRIVATE_POINTER)
-			 || (OMP_CLAUSE_MAP_KIND (c)
-			     == GOMP_MAP_FORCE_DEVICEPTR)))
-		   && t == OMP_CLAUSE_DECL (c)
-		   && !omp_mappable_type (TREE_TYPE (t)))
-	    {
-	      error_at (OMP_CLAUSE_LOCATION (c),
-			"%qD does not have a mappable type in %qs clause", t,
-			omp_clause_code_name[OMP_CLAUSE_CODE (c)]);
-	      remove = true;
-	    }
-	  else if (TREE_TYPE (t) == error_mark_node)
-	    remove = true;
-	  else if (TYPE_ATOMIC (strip_array_types (TREE_TYPE (t))))
-	    {
-	      error_at (OMP_CLAUSE_LOCATION (c),
-			"%<_Atomic%> %qE in %qs clause", t,
-			omp_clause_code_name[OMP_CLAUSE_CODE (c)]);
-	      remove = true;
-	    }
-	  else if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_MAP
-		   && OMP_CLAUSE_MAP_IMPLICIT (c)
-		   && (bitmap_bit_p (&map_head, DECL_UID (t))
-		       || bitmap_bit_p (&map_field_head, DECL_UID (t))
-		       || bitmap_bit_p (&map_firstprivate_head, DECL_UID (t))))
-	    remove = true;
-	  else if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_MAP
-		   && OMP_CLAUSE_MAP_KIND (c) == GOMP_MAP_FIRSTPRIVATE_POINTER)
-	    {
-	      if (bitmap_bit_p (&generic_head, DECL_UID (t))
-		  || bitmap_bit_p (&firstprivate_head, DECL_UID (t))
-		  || bitmap_bit_p (&map_firstprivate_head, DECL_UID (t)))
-		{
-		  error_at (OMP_CLAUSE_LOCATION (c),
-			    "%qD appears more than once in data clauses", t);
+
+		    c_omp_address_inspector ai (OMP_CLAUSE_LOCATION (c), t);
+
+		    if (!omp_parse_expr (addr_tokens, t))
+		      {
+			sorry_at (OMP_CLAUSE_LOCATION (c),
+				  "unsupported map expression %qE",
+				  OMP_CLAUSE_DECL (c));
+			remove = true;
+			break;
+		      }
+
+		    /* This check is to determine if this will be the only map
+		       node created for this clause.  Otherwise, we'll check
+		       the following FIRSTPRIVATE_POINTER or ATTACH_DETACH
+		       node on the next iteration(s) of the loop.   */
+		    if (addr_tokens.length () >= 4
+			&& addr_tokens[0]->type == STRUCTURE_BASE
+			&& addr_tokens[0]->u.structure_base_kind == BASE_DECL
+			&& addr_tokens[1]->type == ACCESS_METHOD
+			&& addr_tokens[2]->type == COMPONENT_SELECTOR
+			&& addr_tokens[3]->type == ACCESS_METHOD
+			&& (addr_tokens[3]->u.access_kind == ACCESS_DIRECT
+			    || (addr_tokens[3]->u.access_kind
+				== ACCESS_INDEXED_ARRAY)))
+		      {
+			tree rt = addr_tokens[1]->expr;
+
+			gcc_assert (DECL_P (rt));
+
+			if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_MAP
+			    && OMP_CLAUSE_MAP_IMPLICIT (c)
+			    && (bitmap_bit_p (&map_head, DECL_UID (rt))
+				|| bitmap_bit_p (&map_field_head, DECL_UID (rt))
+				|| bitmap_bit_p (&map_firstprivate_head,
+						 DECL_UID (rt))))
+			  {
+			    remove = true;
+			    break;
+			  }
+			if (bitmap_bit_p (&map_field_head, DECL_UID (rt)))
+			  break;
+			if (bitmap_bit_p (&map_head, DECL_UID (rt)))
+			  {
+			    if (OMP_CLAUSE_CODE (c) != OMP_CLAUSE_MAP)
+			      error_at (OMP_CLAUSE_LOCATION (c),
+					"%qD appears more than once in motion "
+					"clauses", rt);
+			    else if (openacc)
+			      error_at (OMP_CLAUSE_LOCATION (c),
+					"%qD appears more than once in data "
+					"clauses", rt);
+			    else
+			      error_at (OMP_CLAUSE_LOCATION (c),
+					"%qD appears more than once in map "
+					"clauses", rt);
+			    remove = true;
+			  }
+			else
+			  {
+			    bitmap_set_bit (&map_head, DECL_UID (rt));
+			    bitmap_set_bit (&map_field_head, DECL_UID (rt));
+			  }
+		      }
+		  }
+		if (c_oacc_check_attachments (c))
 		  remove = true;
-		}
-	      else if (bitmap_bit_p (&map_head, DECL_UID (t))
-		       && !bitmap_bit_p (&map_field_head, DECL_UID (t)))
-		{
-		  if (ort == C_ORT_ACC)
+		if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_MAP
+		    && (OMP_CLAUSE_MAP_KIND (c) == GOMP_MAP_ATTACH
+			|| OMP_CLAUSE_MAP_KIND (c) == GOMP_MAP_DETACH)
+		    && !OMP_CLAUSE_SIZE (c))
+		  /* In this case, we have a single array element which is a
+		     pointer, and we already set OMP_CLAUSE_SIZE in
+		     handle_omp_array_sections above.  For attach/detach
+		     clauses, reset the OMP_CLAUSE_SIZE (representing a bias)
+		     to zero here.  */
+		  OMP_CLAUSE_SIZE (c) = size_zero_node;
+		break;
+	      }
+	    else if (!omp_parse_expr (addr_tokens, t))
+	      {
+		sorry_at (OMP_CLAUSE_LOCATION (c),
+			  "unsupported map expression %qE",
+			  OMP_CLAUSE_DECL (c));
+		remove = true;
+		break;
+	      }
+	    if (t == error_mark_node)
+	      {
+		remove = true;
+		break;
+	      }
+	    /* OpenACC attach / detach clauses must be pointers.  */
+	    if (c_oacc_check_attachments (c))
+	      {
+		remove = true;
+		break;
+	      }
+	    if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_MAP
+		&& (OMP_CLAUSE_MAP_KIND (c) == GOMP_MAP_ATTACH
+		    || OMP_CLAUSE_MAP_KIND (c) == GOMP_MAP_DETACH)
+		&& !OMP_CLAUSE_SIZE (c))
+	      /* For attach/detach clauses, set OMP_CLAUSE_SIZE (representing a
+		 bias) to zero here, so it is not set erroneously to the pointer
+		 size later on in gimplify.cc.  */
+	      OMP_CLAUSE_SIZE (c) = size_zero_node;
+
+	    c_omp_address_inspector ai (OMP_CLAUSE_LOCATION (c), t);
+
+	    if (!ai.check_clause (c))
+	      {
+		remove = true;
+		break;
+	      }
+
+	    if (!ai.map_supported_p ())
+	      {
+		sorry_at (OMP_CLAUSE_LOCATION (c),
+			  "unsupported map expression %qE",
+			  OMP_CLAUSE_DECL (c));
+		remove = true;
+		break;
+	      }
+
+	    gcc_assert ((addr_tokens[0]->type == ARRAY_BASE
+			 || addr_tokens[0]->type == STRUCTURE_BASE)
+			&& addr_tokens[1]->type == ACCESS_METHOD);
+
+	    t = addr_tokens[1]->expr;
+
+	    if (addr_tokens[0]->u.structure_base_kind != BASE_DECL)
+	      goto skip_decl_checks;
+
+	    /* For OpenMP, we can access a struct "t" and "t.d" on the same
+	       mapping.  OpenACC allows multiple fields of the same structure
+	       to be written.  */
+	    if (addr_tokens[0]->type == STRUCTURE_BASE
+		&& (bitmap_bit_p (&map_field_head, DECL_UID (t))
+		    || (!openacc && bitmap_bit_p (&map_head, DECL_UID (t)))))
+	      goto skip_decl_checks;
+
+	    if (!VAR_P (t) && TREE_CODE (t) != PARM_DECL)
+	      {
+		error_at (OMP_CLAUSE_LOCATION (c),
+			  "%qE is not a variable in %qs clause", t,
+			  omp_clause_code_name[OMP_CLAUSE_CODE (c)]);
+		remove = true;
+	      }
+	    else if (VAR_P (t) && DECL_THREAD_LOCAL_P (t))
+	      {
+		error_at (OMP_CLAUSE_LOCATION (c),
+			  "%qD is threadprivate variable in %qs clause", t,
+			  omp_clause_code_name[OMP_CLAUSE_CODE (c)]);
+		remove = true;
+	      }
+	    else if ((OMP_CLAUSE_CODE (c) != OMP_CLAUSE_MAP
+		      || (OMP_CLAUSE_MAP_KIND (c)
+			  != GOMP_MAP_FIRSTPRIVATE_POINTER))
+		     && !c_mark_addressable (t))
+	      remove = true;
+	    else if (!(OMP_CLAUSE_CODE (c) == OMP_CLAUSE_MAP
+		       && (OMP_CLAUSE_MAP_KIND (c) == GOMP_MAP_POINTER
+			   || (OMP_CLAUSE_MAP_KIND (c)
+			       == GOMP_MAP_FIRSTPRIVATE_POINTER)
+			   || (OMP_CLAUSE_MAP_KIND (c)
+			       == GOMP_MAP_FORCE_DEVICEPTR)))
+		     && t == OMP_CLAUSE_DECL (c)
+		     && !omp_mappable_type (TREE_TYPE (t)))
+	      {
+		error_at (OMP_CLAUSE_LOCATION (c),
+			  "%qD does not have a mappable type in %qs clause", t,
+			  omp_clause_code_name[OMP_CLAUSE_CODE (c)]);
+		remove = true;
+	      }
+	    else if (TREE_TYPE (t) == error_mark_node)
+	      remove = true;
+	    else if (TYPE_ATOMIC (strip_array_types (TREE_TYPE (t))))
+	      {
+		error_at (OMP_CLAUSE_LOCATION (c),
+			  "%<_Atomic%> %qE in %qs clause", t,
+			  omp_clause_code_name[OMP_CLAUSE_CODE (c)]);
+		remove = true;
+	      }
+	    else if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_MAP
+		     && OMP_CLAUSE_MAP_IMPLICIT (c)
+		     && (bitmap_bit_p (&map_head, DECL_UID (t))
+			 || bitmap_bit_p (&map_field_head, DECL_UID (t))
+			 || bitmap_bit_p (&map_firstprivate_head,
+					  DECL_UID (t))))
+	      remove = true;
+	    else if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_MAP
+		     && (OMP_CLAUSE_MAP_KIND (c)
+			 == GOMP_MAP_FIRSTPRIVATE_POINTER))
+	      {
+		if (bitmap_bit_p (&generic_head, DECL_UID (t))
+		    || bitmap_bit_p (&firstprivate_head, DECL_UID (t))
+		    || bitmap_bit_p (&map_firstprivate_head, DECL_UID (t)))
+		  {
 		    error_at (OMP_CLAUSE_LOCATION (c),
 			      "%qD appears more than once in data clauses", t);
-		  else
+		    remove = true;
+		  }
+		else if (bitmap_bit_p (&map_head, DECL_UID (t))
+			 && !bitmap_bit_p (&map_field_head, DECL_UID (t))
+			 && openacc)
+		  {
 		    error_at (OMP_CLAUSE_LOCATION (c),
-			      "%qD appears both in data and map clauses", t);
-		  remove = true;
-		}
-	      else
-		bitmap_set_bit (&map_firstprivate_head, DECL_UID (t));
-	    }
-	  else if (bitmap_bit_p (&map_head, DECL_UID (t))
-		   && !bitmap_bit_p (&map_field_head, DECL_UID (t)))
-	    {
-	      if (OMP_CLAUSE_CODE (c) != OMP_CLAUSE_MAP)
-		error_at (OMP_CLAUSE_LOCATION (c),
-			  "%qD appears more than once in motion clauses", t);
-	      else if (ort == C_ORT_ACC)
+			      "%qD appears more than once in data clauses", t);
+		    remove = true;
+		  }
+		else
+		  bitmap_set_bit (&map_firstprivate_head, DECL_UID (t));
+	      }
+	    else if (bitmap_bit_p (&map_head, DECL_UID (t))
+		     && !bitmap_bit_p (&map_field_head, DECL_UID (t))
+		     && ort != C_ORT_OMP
+		     && ort != C_ORT_OMP_EXIT_DATA)
+	      {
+		if (OMP_CLAUSE_CODE (c) != OMP_CLAUSE_MAP)
+		  error_at (OMP_CLAUSE_LOCATION (c),
+			    "%qD appears more than once in motion clauses", t);
+		else if (openacc)
+		  error_at (OMP_CLAUSE_LOCATION (c),
+			    "%qD appears more than once in data clauses", t);
+		else
+		  error_at (OMP_CLAUSE_LOCATION (c),
+			    "%qD appears more than once in map clauses", t);
+		remove = true;
+	      }
+	    else if (openacc && bitmap_bit_p (&generic_head, DECL_UID (t)))
+	      {
 		error_at (OMP_CLAUSE_LOCATION (c),
 			  "%qD appears more than once in data clauses", t);
-	      else
-		error_at (OMP_CLAUSE_LOCATION (c),
-			  "%qD appears more than once in map clauses", t);
-	      remove = true;
-	    }
-	  else if (ort == C_ORT_ACC
-		   && bitmap_bit_p (&generic_head, DECL_UID (t)))
-	    {
-	      error_at (OMP_CLAUSE_LOCATION (c),
-			"%qD appears more than once in data clauses", t);
-	      remove = true;
-	    }
-	  else if (bitmap_bit_p (&firstprivate_head, DECL_UID (t))
-		   || bitmap_bit_p (&is_on_device_head, DECL_UID (t)))
-	    {
-	      if (ort == C_ORT_ACC)
-		error_at (OMP_CLAUSE_LOCATION (c),
-			  "%qD appears more than once in data clauses", t);
-	      else
-		error_at (OMP_CLAUSE_LOCATION (c),
-			  "%qD appears both in data and map clauses", t);
-	      remove = true;
-	    }
-	  else
-	    {
-	      bitmap_set_bit (&map_head, DECL_UID (t));
-	      if (t != OMP_CLAUSE_DECL (c)
-		  && TREE_CODE (OMP_CLAUSE_DECL (c)) == COMPONENT_REF)
-		bitmap_set_bit (&map_field_head, DECL_UID (t));
-	    }
+		remove = true;
+	      }
+	    else if (bitmap_bit_p (&firstprivate_head, DECL_UID (t))
+		     || bitmap_bit_p (&is_on_device_head, DECL_UID (t)))
+	      {
+		if (openacc)
+		  error_at (OMP_CLAUSE_LOCATION (c),
+			    "%qD appears more than once in data clauses", t);
+		else
+		  error_at (OMP_CLAUSE_LOCATION (c),
+			    "%qD appears both in data and map clauses", t);
+		remove = true;
+	      }
+	    else if (!omp_access_chain_p (addr_tokens, 1))
+	      {
+		bitmap_set_bit (&map_head, DECL_UID (t));
+		if (t != OMP_CLAUSE_DECL (c)
+		    && TREE_CODE (OMP_CLAUSE_DECL (c)) == COMPONENT_REF)
+		  bitmap_set_bit (&map_field_head, DECL_UID (t));
+	      }
+
+	  skip_decl_checks:
+	    /* If we call omp_expand_map_clause in handle_omp_array_sections,
+	       the containing loop (here) iterates through the new nodes
+	       created by that expansion.  Avoid expanding those again (just
+	       by checking the node type).  */
+	    if (!remove
+		&& ort != C_ORT_DECLARE_SIMD
+		&& (OMP_CLAUSE_CODE (c) != OMP_CLAUSE_MAP
+		    || (OMP_CLAUSE_MAP_KIND (c) != GOMP_MAP_FIRSTPRIVATE_POINTER
+			&& (OMP_CLAUSE_MAP_KIND (c)
+			    != GOMP_MAP_FIRSTPRIVATE_REFERENCE)
+			&& OMP_CLAUSE_MAP_KIND (c) != GOMP_MAP_ALWAYS_POINTER
+			&& OMP_CLAUSE_MAP_KIND (c) != GOMP_MAP_ATTACH_DETACH
+			&& OMP_CLAUSE_MAP_KIND (c) != GOMP_MAP_ATTACH
+			&& OMP_CLAUSE_MAP_KIND (c) != GOMP_MAP_DETACH)))
+	      {
+		grp_start_p = pc;
+		grp_sentinel = OMP_CLAUSE_CHAIN (c);
+		tree nc = ai.expand_map_clause (c, OMP_CLAUSE_DECL (c),
+						addr_tokens, ort);
+		if (nc != error_mark_node)
+		  c = nc;
+	      }
+	  }
 	  break;
 
 	case OMP_CLAUSE_ENTER:
@@ -15709,7 +15653,7 @@ c_finish_omp_clauses (tree clauses, enum c_omp_region_type ort)
 	  if (TREE_CODE (TREE_TYPE (t)) != POINTER_TYPE)
 	    {
 	      if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_USE_DEVICE_PTR
-		  && ort != C_ORT_ACC)
+		  && !openacc)
 		{
 		  error_at (OMP_CLAUSE_LOCATION (c),
 			    "%qs variable is not a pointer",

gcc/cfgexpand.cc

@@ -786,7 +786,13 @@ add_partitioned_vars_to_ptset (struct pt_solution *pt,
 /* Update points-to sets based on partition info, so we can use them on RTL.
    The bitmaps representing stack partitions will be saved until expand,
    where partitioned decls used as bases in memory expressions will be
-   rewritten.  */
+   rewritten.
+
+   It is not necessary to update TBAA info on accesses to the coalesced
+   storage since our memory model doesn't allow TBAA to be used for
+   WAW or WAR dependences.  For RAW when the write is to an old object
+   the new object would not have been initialized at the point of the
+   read, invoking undefined behavior.  */
 
 static void
 update_alias_info_with_stack_vars (void)

gcc/combine.cc

@@ -7466,6 +7466,11 @@ expand_field_assignment (const_rtx x)
       if (!targetm.scalar_mode_supported_p (compute_mode))
 	break;
 
+      /* gen_lowpart_for_combine returns CLOBBER on failure.  */
+      rtx lowpart = gen_lowpart (compute_mode, SET_SRC (x));
+      if (GET_CODE (lowpart) == CLOBBER)
+	break;
+
       /* Now compute the equivalent expression.  Make a copy of INNER
 	 for the SET_DEST in case it is a MEM into which we will substitute;
 	 we don't want shared RTL in that case.  */
@@ -7480,9 +7485,7 @@ expand_field_assignment (const_rtx x)
 				     inner);
       masked = simplify_gen_binary (ASHIFT, compute_mode,
 				    simplify_gen_binary (
-				      AND, compute_mode,
-				      gen_lowpart (compute_mode, SET_SRC (x)),
-				      mask),
+				      AND, compute_mode, lowpart, mask),
 				    pos);
 
       x = gen_rtx_SET (copy_rtx (inner),

gcc/common.opt

@@ -1020,6 +1020,7 @@ Driver Undocumented
 ;
 ; 19: Emits ABI tags if needed in structured binding mangled names.
 ;     Ignores cv-quals on [[no_unique_object]] members.
+;     Mangles constraints on function templates.
 ;     Default in G++ 14.
 ;
 ; Additional positive integers will be assigned as new versions of
@@ -1391,6 +1392,10 @@ Enum(diagnostic_color_rule) String(always) Value(DIAGNOSTICS_COLOR_YES)
 EnumValue
 Enum(diagnostic_color_rule) String(auto) Value(DIAGNOSTICS_COLOR_AUTO)
 
+fdiagnostics-json-formatting
+Common Var(flag_diagnostics_json_formatting) Init(1)
+Enable formatting of JSON output.
+
 fdiagnostics-urls=
 Driver Common Joined RejectNegative Var(flag_diagnostics_show_urls) Enum(diagnostic_url_rule) Init(DIAGNOSTICS_URL_AUTO)
 -fdiagnostics-urls=[never|always|auto]	Embed URLs in diagnostics.

gcc/common/config/aarch64/aarch64-common.cc

@@ -55,6 +55,7 @@ static const struct default_options aarch_option_optimization_table[] =
     { OPT_LEVELS_1_PLUS, OPT_fsched_pressure, NULL, 1 },
     /* Enable redundant extension instructions removal at -O2 and higher.  */
     { OPT_LEVELS_2_PLUS, OPT_free, NULL, 1 },
+    { OPT_LEVELS_2_PLUS, OPT_mearly_ra_, NULL, AARCH64_EARLY_RA_ALL },
 #if (TARGET_DEFAULT_ASYNC_UNWIND_TABLES == 1)
     { OPT_LEVELS_ALL, OPT_fasynchronous_unwind_tables, NULL, 1 },
     { OPT_LEVELS_ALL, OPT_funwind_tables, NULL, 1},

gcc/common/config/riscv/riscv-common.cc

@@ -145,6 +145,15 @@ static const riscv_implied_info_t riscv_implied_info[] =
   {"zvksc", "zvbc"},
   {"zvksg", "zvks"},
   {"zvksg", "zvkg"},
+  {"zvbb",  "zvkb"},
+  {"zvbc",   "zve64x"},
+  {"zvkb",   "zve32x"},
+  {"zvkg",   "zve32x"},
+  {"zvkned", "zve32x"},
+  {"zvknha", "zve32x"},
+  {"zvknhb", "zve64x"},
+  {"zvksed", "zve32x"},
+  {"zvksh",  "zve32x"},
 
   {"zfh", "zfhmin"},
   {"zfhmin", "f"},

gcc/config.gcc

@@ -345,11 +345,11 @@ m32c*-*-*)
         ;;
 aarch64*-*-*)
 	cpu_type=aarch64
-	extra_headers="arm_fp16.h arm_neon.h arm_bf16.h arm_acle.h arm_sve.h arm_sme.h"
+	extra_headers="arm_fp16.h arm_neon.h arm_bf16.h arm_acle.h arm_sve.h arm_sme.h arm_neon_sve_bridge.h"
 	c_target_objs="aarch64-c.o"
 	cxx_target_objs="aarch64-c.o"
 	d_target_objs="aarch64-d.o"
-	extra_objs="aarch64-builtins.o aarch-common.o aarch64-sve-builtins.o aarch64-sve-builtins-shapes.o aarch64-sve-builtins-base.o aarch64-sve-builtins-sve2.o aarch64-sve-builtins-sme.o cortex-a57-fma-steering.o aarch64-speculation.o falkor-tag-collision-avoidance.o aarch-bti-insert.o aarch64-cc-fusion.o"
+	extra_objs="aarch64-builtins.o aarch-common.o aarch64-sve-builtins.o aarch64-sve-builtins-shapes.o aarch64-sve-builtins-base.o aarch64-sve-builtins-sve2.o aarch64-sve-builtins-sme.o cortex-a57-fma-steering.o aarch64-speculation.o falkor-tag-collision-avoidance.o aarch-bti-insert.o aarch64-cc-fusion.o aarch64-early-ra.o"
 	target_gtfiles="\$(srcdir)/config/aarch64/aarch64-builtins.cc \$(srcdir)/config/aarch64/aarch64-sve-builtins.h \$(srcdir)/config/aarch64/aarch64-sve-builtins.cc"
 	target_has_targetm_common=yes
 	;;

gcc/config/aarch64/aarch64-arches.def

@@ -39,10 +39,12 @@ AARCH64_ARCH("armv8.5-a",     generic_armv8_a,   V8_5A,     8,  (V8_4A, SB, SSBS
 AARCH64_ARCH("armv8.6-a",     generic_armv8_a,   V8_6A,     8,  (V8_5A, I8MM, BF16))
 AARCH64_ARCH("armv8.7-a",     generic_armv8_a,   V8_7A,     8,  (V8_6A, LS64))
 AARCH64_ARCH("armv8.8-a",     generic_armv8_a,   V8_8A,     8,  (V8_7A, MOPS))
+AARCH64_ARCH("armv8.9-a",     generic_armv8_a,   V8_9A,     8,  (V8_8A))
 AARCH64_ARCH("armv8-r",       generic_armv8_a,   V8R  ,     8,  (V8_4A))
 AARCH64_ARCH("armv9-a",       generic_armv9_a,   V9A  ,     9,  (V8_5A, SVE2))
 AARCH64_ARCH("armv9.1-a",     generic_armv9_a,   V9_1A,     9,  (V8_6A, V9A))
 AARCH64_ARCH("armv9.2-a",     generic_armv9_a,   V9_2A,     9,  (V8_7A, V9_1A))
 AARCH64_ARCH("armv9.3-a",     generic_armv9_a,   V9_3A,     9,  (V8_8A, V9_2A))
+AARCH64_ARCH("armv9.4-a",     generic_armv9_a,   V9_4A,     9,  (V8_9A, V9_3A))
 
 #undef AARCH64_ARCH

gcc/config/aarch64/aarch64-builtins.cc

@@ -48,6 +48,7 @@
 #include "attribs.h"
 #include "gimple-fold.h"
 #include "builtins.h"
+#include "aarch64-builtins.h"
 
 #define v8qi_UP  E_V8QImode
 #define v8di_UP  E_V8DImode
@@ -184,47 +185,8 @@
 #define SIMD_INTR_QUAL(suffix) QUAL_##suffix
 #define SIMD_INTR_LENGTH_CHAR(length) LENGTH_##length
 
-
 #define SIMD_MAX_BUILTIN_ARGS 5
 
-enum aarch64_type_qualifiers
-{
-  /* T foo.  */
-  qualifier_none = 0x0,
-  /* unsigned T foo.  */
-  qualifier_unsigned = 0x1, /* 1 << 0  */
-  /* const T foo.  */
-  qualifier_const = 0x2, /* 1 << 1  */
-  /* T *foo.  */
-  qualifier_pointer = 0x4, /* 1 << 2  */
-  /* Used when expanding arguments if an operand could
-     be an immediate.  */
-  qualifier_immediate = 0x8, /* 1 << 3  */
-  qualifier_maybe_immediate = 0x10, /* 1 << 4  */
-  /* void foo (...).  */
-  qualifier_void = 0x20, /* 1 << 5  */
-  /* 1 << 6 is now unused */
-  /* Some builtins should use the T_*mode* encoded in a simd_builtin_datum
-     rather than using the type of the operand.  */
-  qualifier_map_mode = 0x80, /* 1 << 7  */
-  /* qualifier_pointer | qualifier_map_mode  */
-  qualifier_pointer_map_mode = 0x84,
-  /* qualifier_const | qualifier_pointer | qualifier_map_mode  */
-  qualifier_const_pointer_map_mode = 0x86,
-  /* Polynomial types.  */
-  qualifier_poly = 0x100,
-  /* Lane indices - must be in range, and flipped for bigendian.  */
-  qualifier_lane_index = 0x200,
-  /* Lane indices for single lane structure loads and stores.  */
-  qualifier_struct_load_store_lane_index = 0x400,
-  /* Lane indices selected in pairs. - must be in range, and flipped for
-     bigendian.  */
-  qualifier_lane_pair_index = 0x800,
-  /* Lane indices selected in quadtuplets. - must be in range, and flipped for
-     bigendian.  */
-  qualifier_lane_quadtup_index = 0x1000,
-};
-
 /* Flags that describe what a function might do.  */
 const unsigned int FLAG_NONE = 0U;
 const unsigned int FLAG_READ_FPCR = 1U << 0;
@@ -815,11 +777,17 @@ enum aarch64_builtins
   AARCH64_RSR64,
   AARCH64_RSRF,
   AARCH64_RSRF64,
+  AARCH64_RSR128,
   AARCH64_WSR,
   AARCH64_WSRP,
   AARCH64_WSR64,
   AARCH64_WSRF,
   AARCH64_WSRF64,
+  AARCH64_WSR128,
+  AARCH64_PLD,
+  AARCH64_PLDX,
+  AARCH64_PLI,
+  AARCH64_PLIX,
   AARCH64_BUILTIN_MAX
 };
 
@@ -895,47 +863,9 @@ const char *aarch64_scalar_builtin_types[] = {
   NULL
 };
 
-#define ENTRY(E, M, Q, G) E,
-enum aarch64_simd_type
-{
-#include "aarch64-simd-builtin-types.def"
-  ARM_NEON_H_TYPES_LAST
-};
-#undef ENTRY
-
-struct GTY(()) aarch64_simd_type_info
-{
-  enum aarch64_simd_type type;
-
-  /* Internal type name.  */
-  const char *name;
-
-  /* Internal type name(mangled).  The mangled names conform to the
-     AAPCS64 (see "Procedure Call Standard for the ARM 64-bit Architecture",
-     Appendix A).  To qualify for emission with the mangled names defined in
-     that document, a vector type must not only be of the correct mode but also
-     be of the correct internal AdvSIMD vector type (e.g. __Int8x8_t); these
-     types are registered by aarch64_init_simd_builtin_types ().  In other
-     words, vector types defined in other ways e.g. via vector_size attribute
-     will get default mangled names.  */
-  const char *mangle;
-
-  /* Internal type.  */
-  tree itype;
-
-  /* Element type.  */
-  tree eltype;
-
-  /* Machine mode the internal type maps to.  */
-  enum machine_mode mode;
-
-  /* Qualifiers.  */
-  enum aarch64_type_qualifiers q;
-};
-
 #define ENTRY(E, M, Q, G)  \
   {E, "__" #E, #G "__" #E, NULL_TREE, NULL_TREE, E_##M##mode, qualifier_##Q},
-static GTY(()) struct aarch64_simd_type_info aarch64_simd_types [] = {
+GTY(()) struct aarch64_simd_type_info aarch64_simd_types [] = {
 #include "aarch64-simd-builtin-types.def"
 };
 #undef ENTRY
@@ -1842,6 +1772,10 @@ aarch64_init_rwsr_builtins (void)
     = build_function_type_list (double_type_node, const_char_ptr_type, NULL);
   AARCH64_INIT_RWSR_BUILTINS_DECL (RSRF64, rsrf64, fntype);
 
+  fntype
+    = build_function_type_list (uint128_type_node, const_char_ptr_type, NULL);
+  AARCH64_INIT_RWSR_BUILTINS_DECL (RSR128, rsr128, fntype);
+
   fntype
     = build_function_type_list (void_type_node, const_char_ptr_type,
 				uint32_type_node, NULL);
@@ -1867,6 +1801,39 @@ aarch64_init_rwsr_builtins (void)
     = build_function_type_list (void_type_node, const_char_ptr_type,
 				double_type_node, NULL);
   AARCH64_INIT_RWSR_BUILTINS_DECL (WSRF64, wsrf64, fntype);
+
+  fntype
+    = build_function_type_list (void_type_node, const_char_ptr_type,
+				uint128_type_node, NULL);
+  AARCH64_INIT_RWSR_BUILTINS_DECL (WSR128, wsr128, fntype);
+}
+
+/* Add builtins for data and instrution prefetch.  */
+static void
+aarch64_init_prefetch_builtin (void)
+{
+#define AARCH64_INIT_PREFETCH_BUILTIN(INDEX, N)				\
+  aarch64_builtin_decls[INDEX] =					\
+    aarch64_general_add_builtin ("__builtin_aarch64_" N, ftype, INDEX)
+
+  tree ftype;
+  tree cv_argtype;
+  cv_argtype = build_qualified_type (void_type_node, TYPE_QUAL_CONST
+						     | TYPE_QUAL_VOLATILE);
+  cv_argtype = build_pointer_type (cv_argtype);
+
+  ftype = build_function_type_list (void_type_node, cv_argtype, NULL);
+  AARCH64_INIT_PREFETCH_BUILTIN (AARCH64_PLD, "pld");
+  AARCH64_INIT_PREFETCH_BUILTIN (AARCH64_PLI, "pli");
+
+  ftype = build_function_type_list (void_type_node, unsigned_type_node,
+				    unsigned_type_node, unsigned_type_node,
+				    cv_argtype, NULL);
+  AARCH64_INIT_PREFETCH_BUILTIN (AARCH64_PLDX, "pldx");
+
+  ftype = build_function_type_list (void_type_node, unsigned_type_node,
+				    unsigned_type_node, cv_argtype, NULL);
+  AARCH64_INIT_PREFETCH_BUILTIN (AARCH64_PLIX, "plix");
 }
 
 /* Initialize the memory tagging extension (MTE) builtins.  */
@@ -2091,6 +2058,7 @@ aarch64_general_init_builtins (void)
   aarch64_init_data_intrinsics ();
 
   aarch64_init_rwsr_builtins ();
+  aarch64_init_prefetch_builtin ();
 
   tree ftype_jcvt
     = build_function_type_list (intSI_type_node, double_type_node, NULL);
@@ -2710,6 +2678,7 @@ aarch64_expand_rwsr_builtin (tree exp, rtx target, int fcode)
   tree arg0, arg1;
   rtx const_str, input_val, subreg;
   enum machine_mode mode;
+  enum insn_code icode;
   class expand_operand ops[2];
 
   arg0 = CALL_EXPR_ARG (exp, 0);
@@ -2718,7 +2687,18 @@ aarch64_expand_rwsr_builtin (tree exp, rtx target, int fcode)
 		   || fcode == AARCH64_WSRP
 		   || fcode == AARCH64_WSR64
 		   || fcode == AARCH64_WSRF
-		   || fcode == AARCH64_WSRF64);
+		   || fcode == AARCH64_WSRF64
+		   || fcode == AARCH64_WSR128);
+
+  bool op128 = (fcode == AARCH64_RSR128 || fcode == AARCH64_WSR128);
+  enum machine_mode sysreg_mode = op128 ? TImode : DImode;
+
+  if (op128 && !TARGET_D128)
+    {
+      error_at (EXPR_LOCATION (exp), "128-bit system register support requires"
+				     " the %<d128%> extension");
+      return const0_rtx;
+    }
 
   /* Argument 0 (system register name) must be a string literal.  */
   gcc_assert (TREE_CODE (arg0) == ADDR_EXPR
@@ -2740,7 +2720,8 @@ aarch64_expand_rwsr_builtin (tree exp, rtx target, int fcode)
   for (unsigned pos = 0; pos <= len; pos++)
     sysreg_name[pos] = TOLOWER (sysreg_name[pos]);
 
-  const char *name_output = aarch64_retrieve_sysreg (sysreg_name, write_op);
+  const char* name_output = aarch64_retrieve_sysreg ((const char *) sysreg_name,
+						     write_op, op128);
   if (name_output == NULL)
     {
       error_at (EXPR_LOCATION (exp), "invalid system register name %qs",
@@ -2760,13 +2741,17 @@ aarch64_expand_rwsr_builtin (tree exp, rtx target, int fcode)
       mode = TYPE_MODE (TREE_TYPE (arg1));
       input_val = copy_to_mode_reg (mode, expand_normal (arg1));
 
+      icode = (op128 ? CODE_FOR_aarch64_write_sysregti
+		     : CODE_FOR_aarch64_write_sysregdi);
+
       switch (fcode)
 	{
 	case AARCH64_WSR:
 	case AARCH64_WSRP:
 	case AARCH64_WSR64:
 	case AARCH64_WSRF64:
-	  subreg = lowpart_subreg (DImode, input_val, mode);
+	case AARCH64_WSR128:
+	  subreg = lowpart_subreg (sysreg_mode, input_val, mode);
 	  break;
 	case AARCH64_WSRF:
 	  subreg = gen_lowpart_SUBREG (SImode, input_val);
@@ -2775,8 +2760,8 @@ aarch64_expand_rwsr_builtin (tree exp, rtx target, int fcode)
 	}
 
       create_fixed_operand (&ops[0], const_str);
-      create_input_operand (&ops[1], subreg, DImode);
-      expand_insn (CODE_FOR_aarch64_write_sysregdi, 2, ops);
+      create_input_operand (&ops[1], subreg, sysreg_mode);
+      expand_insn (icode, 2, ops);
 
       return target;
     }
@@ -2784,10 +2769,13 @@ aarch64_expand_rwsr_builtin (tree exp, rtx target, int fcode)
   /* Read operations are implied by !write_op.  */
   gcc_assert (call_expr_nargs (exp) == 1);
 
+  icode = (op128 ? CODE_FOR_aarch64_read_sysregti
+		 : CODE_FOR_aarch64_read_sysregdi);
+
   /* Emit the initial read_sysregdi rtx.  */
-  create_output_operand (&ops[0], target, DImode);
+  create_output_operand (&ops[0], target, sysreg_mode);
   create_fixed_operand (&ops[1], const_str);
-  expand_insn (CODE_FOR_aarch64_read_sysregdi, 2, ops);
+  expand_insn (icode, 2, ops);
   target = ops[0].value;
 
   /* Do any necessary post-processing on the result.  */
@@ -2797,7 +2785,8 @@ aarch64_expand_rwsr_builtin (tree exp, rtx target, int fcode)
     case AARCH64_RSRP:
     case AARCH64_RSR64:
     case AARCH64_RSRF64:
-      return lowpart_subreg (TYPE_MODE (TREE_TYPE (exp)), target, DImode);
+    case AARCH64_RSR128:
+      return lowpart_subreg (TYPE_MODE (TREE_TYPE (exp)), target, sysreg_mode);
     case AARCH64_RSRF:
       subreg = gen_lowpart_SUBREG (SImode, target);
       return gen_lowpart_SUBREG (SFmode, subreg);
@@ -2806,6 +2795,93 @@ aarch64_expand_rwsr_builtin (tree exp, rtx target, int fcode)
     }
 }
 
+/* Ensure argument ARGNO in EXP represents a const-type argument in the range
+   [MINVAL, MAXVAL).  */
+static HOST_WIDE_INT
+require_const_argument (tree exp, unsigned int argno, HOST_WIDE_INT minval,
+			HOST_WIDE_INT maxval)
+{
+  maxval--;
+  tree arg = CALL_EXPR_ARG (exp, argno);
+  if (TREE_CODE (arg) != INTEGER_CST)
+      error_at (EXPR_LOCATION (exp), "Constant-type argument expected");
+
+  auto argval = wi::to_widest (arg);
+
+  if (argval < minval || argval > maxval)
+    error_at (EXPR_LOCATION (exp),
+	      "argument %d must be a constant immediate "
+	      "in range [%wd,%wd]", argno + 1, minval, maxval);
+
+  HOST_WIDE_INT retval = argval.to_shwi ();
+  return retval;
+}
+
+
+/* Expand a prefetch builtin EXP.  */
+void
+aarch64_expand_prefetch_builtin (tree exp, int fcode)
+{
+  int kind_id = -1;
+  int level_id = -1;
+  int rettn_id = -1;
+  char prfop[11];
+  class expand_operand ops[2];
+
+  static const char *kind_s[] = {"PLD", "PST", "PLI"};
+  static const char *level_s[] = {"L1", "L2", "L3", "SLC"};
+  static const char *rettn_s[] = {"KEEP", "STRM"};
+
+  /* Each of the four prefetch builtins takes a different number of arguments,
+     but proceeds to call the PRFM insn which requires 4 pieces of information
+     to be fully defined.  Where one of these takes less than 4 arguments, set
+     sensible defaults.  */
+  switch (fcode)
+    {
+    case AARCH64_PLDX:
+      break;
+    case AARCH64_PLIX:
+      kind_id = 2;
+      break;
+    case AARCH64_PLI:
+    case AARCH64_PLD:
+      kind_id  = (fcode == AARCH64_PLD) ? 0 : 2;
+      level_id = 0;
+      rettn_id = 0;
+      break;
+    default:
+      gcc_unreachable ();
+    }
+
+  /* Any -1 id variable is to be user-supplied.  Here we fill these in and run
+     bounds checks on them.  "PLI" is used only implicitly by AARCH64_PLI &
+     AARCH64_PLIX, never explicitly.  */
+  int argno = 0;
+  if (kind_id < 0)
+    kind_id = require_const_argument (exp, argno++, 0, ARRAY_SIZE (kind_s) - 1);
+  if (level_id < 0)
+    level_id = require_const_argument (exp, argno++, 0, ARRAY_SIZE (level_s));
+  if (rettn_id < 0)
+    rettn_id = require_const_argument (exp, argno++, 0, ARRAY_SIZE (rettn_s));
+  rtx address = expand_expr (CALL_EXPR_ARG (exp, argno), NULL_RTX, Pmode,
+			     EXPAND_NORMAL);
+
+  if (seen_error ())
+    return;
+
+  sprintf (prfop, "%s%s%s", kind_s[kind_id],
+			    level_s[level_id],
+			    rettn_s[rettn_id]);
+
+  rtx const_str = rtx_alloc (CONST_STRING);
+  PUT_CODE (const_str, CONST_STRING);
+  XSTR (const_str, 0) = ggc_strdup (prfop);
+
+  create_fixed_operand (&ops[0], const_str);
+  create_address_operand (&ops[1], address);
+  maybe_expand_insn (CODE_FOR_aarch64_pldx, 2, ops);
+}
+
 /* Expand an expression EXP that calls a MEMTAG built-in FCODE
    with result going to TARGET.  */
 static rtx
@@ -3044,12 +3120,20 @@ aarch64_general_expand_builtin (unsigned int fcode, tree exp, rtx target,
     case AARCH64_RSR64:
     case AARCH64_RSRF:
     case AARCH64_RSRF64:
+    case AARCH64_RSR128:
     case AARCH64_WSR:
     case AARCH64_WSRP:
     case AARCH64_WSR64:
     case AARCH64_WSRF:
     case AARCH64_WSRF64:
+    case AARCH64_WSR128:
       return aarch64_expand_rwsr_builtin (exp, target, fcode);
+    case AARCH64_PLD:
+    case AARCH64_PLDX:
+    case AARCH64_PLI:
+    case AARCH64_PLIX:
+      aarch64_expand_prefetch_builtin (exp, fcode);
+      return target;
     }
 
   if (fcode >= AARCH64_SIMD_BUILTIN_BASE && fcode <= AARCH64_SIMD_BUILTIN_MAX)

gcc/config/aarch64/aarch64-builtins.h

@@ -0,0 +1,99 @@
+/* Builtins' description for AArch64 SIMD architecture.
+   Copyright (C) 2023 Free Software Foundation, Inc.
+   This file is part of GCC.
+
+   GCC is free software; you can redistribute it and/or modify it
+   under the terms of the GNU General Public License as published by
+   the Free Software Foundation; either version 3, or (at your option)
+   any later version.
+
+   GCC is distributed in the hope that it will be useful, but
+   WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+   General Public License for more details.
+
+   You should have received a copy of the GNU General Public License
+   along with GCC; see the file COPYING3.  If not see
+   <http://www.gnu.org/licenses/>.  */
+#ifndef GCC_AARCH64_BUILTINS_H
+#define GCC_AARCH64_BUILTINS_H
+
+enum aarch64_type_qualifiers
+{
+  /* T foo.  */
+  qualifier_none = 0x0,
+  /* unsigned T foo.  */
+  qualifier_unsigned = 0x1, /* 1 << 0  */
+  /* const T foo.  */
+  qualifier_const = 0x2, /* 1 << 1  */
+  /* T *foo.  */
+  qualifier_pointer = 0x4, /* 1 << 2  */
+  /* Used when expanding arguments if an operand could
+     be an immediate.  */
+  qualifier_immediate = 0x8, /* 1 << 3  */
+  qualifier_maybe_immediate = 0x10, /* 1 << 4  */
+  /* void foo (...).  */
+  qualifier_void = 0x20, /* 1 << 5  */
+  /* 1 << 6 is now unused */
+  /* Some builtins should use the T_*mode* encoded in a simd_builtin_datum
+     rather than using the type of the operand.  */
+  qualifier_map_mode = 0x80, /* 1 << 7  */
+  /* qualifier_pointer | qualifier_map_mode  */
+  qualifier_pointer_map_mode = 0x84,
+  /* qualifier_const | qualifier_pointer | qualifier_map_mode  */
+  qualifier_const_pointer_map_mode = 0x86,
+  /* Polynomial types.  */
+  qualifier_poly = 0x100,
+  /* Lane indices - must be in range, and flipped for bigendian.  */
+  qualifier_lane_index = 0x200,
+  /* Lane indices for single lane structure loads and stores.  */
+  qualifier_struct_load_store_lane_index = 0x400,
+  /* Lane indices selected in pairs. - must be in range, and flipped for
+     bigendian.  */
+  qualifier_lane_pair_index = 0x800,
+  /* Lane indices selected in quadtuplets. - must be in range, and flipped for
+     bigendian.  */
+  qualifier_lane_quadtup_index = 0x1000,
+};
+
+#define ENTRY(E, M, Q, G) E,
+enum aarch64_simd_type
+{
+#include "aarch64-simd-builtin-types.def"
+  ARM_NEON_H_TYPES_LAST
+};
+#undef ENTRY
+
+struct GTY(()) aarch64_simd_type_info
+{
+  enum aarch64_simd_type type;
+
+  /* Internal type name.  */
+  const char *name;
+
+  /* Internal type name(mangled).  The mangled names conform to the
+     AAPCS64 (see "Procedure Call Standard for the ARM 64-bit Architecture",
+     Appendix A).  To qualify for emission with the mangled names defined in
+     that document, a vector type must not only be of the correct mode but also
+     be of the correct internal AdvSIMD vector type (e.g. __Int8x8_t); these
+     types are registered by aarch64_init_simd_builtin_types ().  In other
+     words, vector types defined in other ways e.g. via vector_size attribute
+     will get default mangled names.  */
+  const char *mangle;
+
+  /* Internal type.  */
+  tree itype;
+
+  /* Element type.  */
+  tree eltype;
+
+  /* Machine mode the internal type maps to.  */
+  enum machine_mode mode;
+
+  /* Qualifiers.  */
+  enum aarch64_type_qualifiers q;
+};
+
+extern aarch64_simd_type_info aarch64_simd_types[];
+
+#endif

gcc/config/aarch64/aarch64-c.cc

@@ -254,6 +254,7 @@ aarch64_update_cpp_builtins (cpp_reader *pfile)
   aarch64_def_or_undef (TARGET_LS64,
 			"__ARM_FEATURE_LS64", pfile);
   aarch64_def_or_undef (AARCH64_ISA_RCPC, "__ARM_FEATURE_RCPC", pfile);
+  aarch64_def_or_undef (TARGET_D128, "__ARM_FEATURE_SYSREG128", pfile);
 
   aarch64_def_or_undef (TARGET_SME, "__ARM_FEATURE_SME", pfile);
   aarch64_def_or_undef (TARGET_SME_I16I64, "__ARM_FEATURE_SME_I16I64", pfile);
@@ -350,6 +351,8 @@ aarch64_pragma_aarch64 (cpp_reader *)
     handle_arm_neon_h ();
   else if (strcmp (name, "arm_acle.h") == 0)
     handle_arm_acle_h ();
+  else if (strcmp (name, "arm_neon_sve_bridge.h") == 0)
+    aarch64_sve::handle_arm_neon_sve_bridge_h ();
   else
     error ("unknown %<#pragma GCC aarch64%> option %qs", name);
 }

gcc/config/aarch64/aarch64-neon-sve-bridge-builtins.def

@@ -0,0 +1,28 @@
+/* Builtin lists for AArch64 NEON-SVE-Bridge
+   Copyright (C) 2023 Free Software Foundation, Inc.
+
+   This file is part of GCC.
+
+   GCC is free software; you can redistribute it and/or modify it
+   under the terms of the GNU General Public License as published by
+   the Free Software Foundation; either version 3, or (at your option)
+   any later version.
+
+   GCC is distributed in the hope that it will be useful, but
+   WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+   General Public License for more details.
+
+   You should have received a copy of the GNU General Public License
+   along with GCC; see the file COPYING3.  If not see
+   <http://www.gnu.org/licenses/>.  */
+
+#ifndef DEF_NEON_SVE_FUNCTION
+#define DEF_NEON_SVE_FUNCTION(A, B, C, D, E)
+#endif
+
+DEF_NEON_SVE_FUNCTION (svset_neonq, set_neonq, all_data, none, none)
+DEF_NEON_SVE_FUNCTION (svget_neonq, get_neonq, all_data, none, none)
+DEF_NEON_SVE_FUNCTION (svdup_neonq, dup_neonq, all_data, none, none)
+
+#undef DEF_NEON_SVE_FUNCTION

gcc/config/aarch64/aarch64-option-extensions.def

@@ -159,4 +159,11 @@ AARCH64_OPT_EXTENSION("sme-f64f64", SME_F64F64, (SME), (), (), "")
 
 AARCH64_OPT_EXTENSION("sme2", SME2, (SME), (), (), "sme2")
 
+AARCH64_OPT_EXTENSION("d128", D128, (), (), (), "d128")
+
+AARCH64_OPT_EXTENSION("the", THE, (), (), (), "the")
+
+AARCH64_OPT_EXTENSION("gcs", GCS, (), (), (), "gcs")
+
+AARCH64_OPT_EXTENSION("rcpc3", RCPC3, (), (), (), "rcpc3")
 #undef AARCH64_OPT_EXTENSION

gcc/config/aarch64/aarch64-opts.h

@@ -120,4 +120,15 @@ enum aarch64_ldp_stp_policy {
   AARCH64_LDP_STP_POLICY_NEVER
 };
 
+/* An enum specifying when the early-ra pass should be run:
+   - AARCH64_EARLY_RA_ALL: for all functions
+   - AARCH64_EARLY_RA_STRIDED: for functions that have access to strided
+     multi-register instructions
+   - AARCH64_EARLY_RA_NONE: for no functions.  */
+enum aarch64_early_ra_scope {
+  AARCH64_EARLY_RA_ALL,
+  AARCH64_EARLY_RA_STRIDED,
+  AARCH64_EARLY_RA_NONE
+};
+
 #endif

gcc/config/aarch64/aarch64-passes.def

@@ -18,6 +18,7 @@
    along with GCC; see the file COPYING3.  If not see
    <http://www.gnu.org/licenses/>.  */
 
+INSERT_PASS_BEFORE (pass_sched, 1, pass_aarch64_early_ra);
 INSERT_PASS_AFTER (pass_regrename, 1, pass_fma_steering);
 INSERT_PASS_BEFORE (pass_reorder_blocks, 1, pass_track_speculation);
 INSERT_PASS_BEFORE (pass_late_thread_prologue_and_epilogue, 1, pass_switch_pstate_sm);

gcc/config/aarch64/aarch64-protos.h

@@ -789,6 +789,7 @@ bool aarch64_mask_and_shift_for_ubfiz_p (scalar_int_mode, rtx, rtx);
 bool aarch64_masks_and_shift_for_bfi_p (scalar_int_mode, unsigned HOST_WIDE_INT,
 					unsigned HOST_WIDE_INT,
 					unsigned HOST_WIDE_INT);
+rtx aarch64_sve_reinterpret (machine_mode, rtx);
 bool aarch64_zero_extend_const_eq (machine_mode, rtx, machine_mode, rtx);
 bool aarch64_move_imm (unsigned HOST_WIDE_INT, machine_mode);
 machine_mode aarch64_sve_int_mode (machine_mode);
@@ -840,7 +841,7 @@ bool aarch64_sve_ptrue_svpattern_p (rtx, struct simd_immediate_info *);
 bool aarch64_simd_valid_immediate (rtx, struct simd_immediate_info *,
 			enum simd_immediate_check w = AARCH64_CHECK_MOV);
 bool aarch64_valid_sysreg_name_p (const char *);
-const char *aarch64_retrieve_sysreg (const char *, bool);
+const char *aarch64_retrieve_sysreg (const char *, bool, bool);
 rtx aarch64_check_zero_based_sve_index_immediate (rtx);
 bool aarch64_maybe_generate_simd_constant (rtx, rtx, machine_mode);
 bool aarch64_simd_special_constant_p (rtx, machine_mode);
@@ -891,6 +892,7 @@ bool aarch64_sme_ldr_vnum_offset_p (rtx, rtx);
 rtx aarch64_simd_vect_par_cnst_half (machine_mode, int, bool);
 rtx aarch64_gen_stepped_int_parallel (unsigned int, int, int);
 bool aarch64_stepped_int_parallel_p (rtx, int);
+bool aarch64_strided_registers_p (rtx *, unsigned int, unsigned int);
 rtx aarch64_tls_get_addr (void);
 unsigned aarch64_debugger_regno (unsigned);
 unsigned aarch64_trampoline_size (void);
@@ -1010,6 +1012,7 @@ namespace aarch64_sve {
   void init_builtins ();
   void handle_arm_sve_h ();
   void handle_arm_sme_h ();
+  void handle_arm_neon_sve_bridge_h ();
   tree builtin_decl (unsigned, bool);
   bool builtin_type_p (const_tree);
   bool builtin_type_p (const_tree, unsigned int *, unsigned int *);
@@ -1063,6 +1066,7 @@ void aarch64_get_all_extension_candidates (auto_vec<const char *> *candidates);
 std::string aarch64_get_extension_string_for_isa_flags (aarch64_feature_flags,
 							aarch64_feature_flags);
 
+rtl_opt_pass *make_pass_aarch64_early_ra (gcc::context *);
 rtl_opt_pass *make_pass_fma_steering (gcc::context *);
 rtl_opt_pass *make_pass_track_speculation (gcc::context *);
 rtl_opt_pass *make_pass_tag_collision_avoidance (gcc::context *);

gcc/config/aarch64/aarch64-simd-builtins.def

@@ -43,6 +43,13 @@
    help describe the attributes (for example, pure) for the intrinsic
    function.  */
 
+  BUILTIN_V12DIF (LOADSTRUCT_LANE, vec_ldap1_lane, 0, ALL)
+  BUILTIN_V12DI (LOADSTRUCT_LANE_U, vec_ldap1_lane, 0, ALL)
+  BUILTIN_V12DI (LOADSTRUCT_LANE_P, vec_ldap1_lane, 0, ALL)
+  BUILTIN_V12DIF (STORESTRUCT_LANE, vec_stl1_lane, 0, ALL)
+  BUILTIN_V12DI (STORESTRUCT_LANE_U, vec_stl1_lane, 0, ALL)
+  BUILTIN_V12DI (STORESTRUCT_LANE_P, vec_stl1_lane, 0, ALL)
+
   BUILTIN_VDC (BINOP, combine, 0, AUTO_FP)
   BUILTIN_VD_I (BINOPU, combine, 0, NONE)
   BUILTIN_VDC_P (BINOPP, combine, 0, NONE)

gcc/config/aarch64/aarch64-simd.md

@@ -7828,6 +7828,71 @@
   DONE;
 })
 
+;; Patterns for rcpc3 vector lane loads and stores.
+
+(define_insn "aarch64_vec_stl1_lanes<mode>_lane<Vel>"
+  [(set (match_operand:BLK 0 "aarch64_simd_struct_operand" "=Q")
+	(unspec:BLK [(match_operand:V12DIF 1 "register_operand" "w")
+		     (match_operand:SI 2 "immediate_operand" "i")]
+		     UNSPEC_STL1_LANE))]
+  "TARGET_RCPC3"
+  {
+    operands[2] = aarch64_endian_lane_rtx (<MODE>mode,
+					   INTVAL (operands[2]));
+    return "stl1\\t{%S1.<Vetype>}[%2], %0";
+  }
+  [(set_attr "type" "neon_store2_one_lane")]
+)
+
+(define_expand "aarch64_vec_stl1_lane<mode>"
+ [(match_operand:DI 0 "register_operand")
+  (match_operand:V12DIF 1 "register_operand")
+  (match_operand:SI 2 "immediate_operand")]
+  "TARGET_RCPC3"
+{
+  rtx mem = gen_rtx_MEM (BLKmode, operands[0]);
+  set_mem_size (mem, GET_MODE_SIZE (GET_MODE_INNER (<MODE>mode)));
+
+  aarch64_simd_lane_bounds (operands[2], 0,
+			    GET_MODE_NUNITS (<MODE>mode).to_constant (), NULL);
+  emit_insn (gen_aarch64_vec_stl1_lanes<mode>_lane<Vel> (mem,
+					operands[1], operands[2]));
+  DONE;
+})
+
+(define_insn "aarch64_vec_ldap1_lanes<mode>_lane<Vel>"
+  [(set (match_operand:V12DIF 0 "register_operand" "=w")
+	(unspec:V12DIF [
+		(match_operand:BLK 1 "aarch64_simd_struct_operand" "Q")
+		(match_operand:V12DIF 2 "register_operand" "0")
+		(match_operand:SI 3 "immediate_operand" "i")]
+		UNSPEC_LDAP1_LANE))]
+  "TARGET_RCPC3"
+  {
+    operands[3] = aarch64_endian_lane_rtx (<MODE>mode,
+					   INTVAL (operands[3]));
+    return "ldap1\\t{%S0.<Vetype>}[%3], %1";
+  }
+  [(set_attr "type" "neon_load2_one_lane")]
+)
+
+(define_expand "aarch64_vec_ldap1_lane<mode>"
+  [(match_operand:V12DIF 0 "register_operand")
+	(match_operand:DI 1 "register_operand")
+	(match_operand:V12DIF 2 "register_operand")
+	(match_operand:SI 3 "immediate_operand")]
+  "TARGET_RCPC3"
+{
+  rtx mem = gen_rtx_MEM (BLKmode, operands[1]);
+  set_mem_size (mem, GET_MODE_SIZE (GET_MODE_INNER (<MODE>mode)));
+
+  aarch64_simd_lane_bounds (operands[3], 0,
+			    GET_MODE_NUNITS (<MODE>mode).to_constant (), NULL);
+  emit_insn (gen_aarch64_vec_ldap1_lanes<mode>_lane<Vel> (operands[0],
+				mem, operands[2], operands[3]));
+  DONE;
+})
+
 (define_insn_and_split "aarch64_rev_reglist<mode>"
 [(set (match_operand:VSTRUCT_QD 0 "register_operand" "=&w")
 	(unspec:VSTRUCT_QD

gcc/config/aarch64/aarch64-sme.md

@@ -1981,4 +1981,74 @@
   "TARGET_STREAMING_SME2
    && !(<LUTI_BITS> == 4 && <vector_count> == 4 && <elem_bits> == 8)"
   "luti<LUTI_BITS>\t%0, zt0, %1[%2]"
+  [(set_attr "stride_type" "luti_consecutive")]
+)
+
+(define_insn "@aarch64_sme_lut<LUTI_BITS><mode>_strided2"
+  [(set (match_operand:SVE_FULL_BHS 0 "aarch64_simd_register" "=Uwd")
+	(unspec:SVE_FULL_BHS
+	  [(reg:V8DI ZT0_REGNUM)
+	   (reg:DI SME_STATE_REGNUM)
+	   (match_operand:VNx16QI 2 "register_operand" "w")
+	   (match_operand:DI 3 "const_int_operand")
+	   (const_int LUTI_BITS)
+	   (const_int 0)]
+	  UNSPEC_SME_LUTI))
+   (set (match_operand:SVE_FULL_BHS 1 "aarch64_simd_register" "=w")
+	(unspec:SVE_FULL_BHS
+	  [(reg:V8DI ZT0_REGNUM)
+	   (reg:DI SME_STATE_REGNUM)
+	   (match_dup 2)
+	   (match_dup 3)
+	   (const_int LUTI_BITS)
+	   (const_int 1)]
+	  UNSPEC_SME_LUTI))]
+  "TARGET_STREAMING_SME2
+   && aarch64_strided_registers_p (operands, 2, 8)"
+  "luti<LUTI_BITS>\t{%0.<Vetype>, %1.<Vetype>}, zt0, %2[%3]"
+  [(set_attr "stride_type" "luti_strided")]
+)
+
+(define_insn "@aarch64_sme_lut<LUTI_BITS><mode>_strided4"
+  [(set (match_operand:SVE_FULL_BHS 0 "aarch64_simd_register" "=Uwt")
+	(unspec:SVE_FULL_BHS
+	  [(reg:V8DI ZT0_REGNUM)
+	   (reg:DI SME_STATE_REGNUM)
+	   (match_operand:VNx16QI 4 "register_operand" "w")
+	   (match_operand:DI 5 "const_int_operand")
+	   (const_int LUTI_BITS)
+	   (const_int 0)]
+	  UNSPEC_SME_LUTI))
+   (set (match_operand:SVE_FULL_BHS 1 "aarch64_simd_register" "=w")
+	(unspec:SVE_FULL_BHS
+	  [(reg:V8DI ZT0_REGNUM)
+	   (reg:DI SME_STATE_REGNUM)
+	   (match_dup 4)
+	   (match_dup 5)
+	   (const_int LUTI_BITS)
+	   (const_int 1)]
+	  UNSPEC_SME_LUTI))
+   (set (match_operand:SVE_FULL_BHS 2 "aarch64_simd_register" "=w")
+	(unspec:SVE_FULL_BHS
+	  [(reg:V8DI ZT0_REGNUM)
+	   (reg:DI SME_STATE_REGNUM)
+	   (match_dup 4)
+	   (match_dup 5)
+	   (const_int LUTI_BITS)
+	   (const_int 2)]
+	  UNSPEC_SME_LUTI))
+   (set (match_operand:SVE_FULL_BHS 3 "aarch64_simd_register" "=w")
+	(unspec:SVE_FULL_BHS
+	  [(reg:V8DI ZT0_REGNUM)
+	   (reg:DI SME_STATE_REGNUM)
+	   (match_dup 4)
+	   (match_dup 5)
+	   (const_int LUTI_BITS)
+	   (const_int 3)]
+	  UNSPEC_SME_LUTI))]
+  "TARGET_STREAMING_SME2
+   && !(<LUTI_BITS> == 4 && <elem_bits> == 8)
+   && aarch64_strided_registers_p (operands, 4, 4)"
+  "luti<LUTI_BITS>\t{%0.<Vetype>, %1.<Vetype>, %2.<Vetype>, %3.<Vetype>}, zt0, %4[%5]"
+  [(set_attr "stride_type" "luti_strided")]
 )

gcc/config/aarch64/aarch64-sve-builtins-base.cc

@@ -44,6 +44,7 @@
 #include "aarch64-sve-builtins-shapes.h"
 #include "aarch64-sve-builtins-base.h"
 #include "aarch64-sve-builtins-functions.h"
+#include "aarch64-builtins.h"
 #include "ssa.h"
 #include "gimple-fold.h"
 
@@ -1099,6 +1100,112 @@ public:
   }
 };
 
+class svget_neonq_impl : public function_base
+{
+public:
+  gimple *
+  fold (gimple_folder &f) const override
+  {
+    if (BYTES_BIG_ENDIAN)
+      return NULL;
+    tree rhs_sve_vector = gimple_call_arg (f.call, 0);
+    tree rhs_vector = build3 (BIT_FIELD_REF, TREE_TYPE (f.lhs),
+			     rhs_sve_vector, bitsize_int (128), bitsize_int (0));
+    return gimple_build_assign (f.lhs, rhs_vector);
+  }
+
+  rtx
+  expand (function_expander &e) const override
+  {
+    if (BYTES_BIG_ENDIAN)
+      {
+	machine_mode mode = e.vector_mode (0);
+	insn_code icode = code_for_aarch64_sve_get_neonq (mode);
+	unsigned int nunits = 128 / GET_MODE_UNIT_BITSIZE (mode);
+	rtx indices = aarch64_gen_stepped_int_parallel
+	  (nunits, nunits - 1, -1);
+
+	e.add_output_operand (icode);
+	e.add_input_operand (icode, e.args[0]);
+	e.add_fixed_operand (indices);
+	return e.generate_insn (icode);
+      }
+    return simplify_gen_subreg (e.result_mode (), e.args[0],
+				GET_MODE (e.args[0]), 0);
+  }
+};
+
+class svset_neonq_impl : public function_base
+{
+public:
+  rtx
+  expand (function_expander &e) const override
+  {
+    machine_mode mode = e.vector_mode (0);
+    rtx_vector_builder builder (VNx16BImode, 16, 2);
+    for (unsigned int i = 0; i < 16; i++)
+      builder.quick_push (CONST1_RTX (BImode));
+    for (unsigned int i = 0; i < 16; i++)
+      builder.quick_push (CONST0_RTX (BImode));
+    e.args.quick_push (builder.build ());
+    if (BYTES_BIG_ENDIAN)
+      return e.use_exact_insn (code_for_aarch64_sve_set_neonq (mode));
+    insn_code icode = code_for_vcond_mask (mode, mode);
+    e.args[1] = lowpart_subreg (mode, e.args[1], GET_MODE (e.args[1]));
+    e.add_output_operand (icode);
+    e.add_input_operand (icode, e.args[1]);
+    e.add_input_operand (icode, e.args[0]);
+    e.add_input_operand (icode, e.args[2]);
+    return e.generate_insn (icode);
+  }
+};
+
+class svdup_neonq_impl : public function_base
+{
+public:
+  gimple *
+  fold (gimple_folder &f) const override
+  {
+    if (BYTES_BIG_ENDIAN)
+      return NULL;
+    tree rhs_vector = gimple_call_arg (f.call, 0);
+    unsigned HOST_WIDE_INT neon_nelts
+      = TYPE_VECTOR_SUBPARTS (TREE_TYPE (rhs_vector)).to_constant ();
+    poly_uint64 sve_nelts = TYPE_VECTOR_SUBPARTS (TREE_TYPE (f.lhs));
+    vec_perm_builder builder (sve_nelts, neon_nelts, 1);
+    for (unsigned int i = 0; i < neon_nelts; i++)
+      builder.quick_push (i);
+    vec_perm_indices indices (builder, 1, neon_nelts);
+    tree perm_type = build_vector_type (ssizetype, sve_nelts);
+    return gimple_build_assign (f.lhs, VEC_PERM_EXPR,
+				rhs_vector,
+				rhs_vector,
+				vec_perm_indices_to_tree (perm_type, indices));
+  }
+
+  rtx
+  expand (function_expander &e) const override
+  {
+    machine_mode mode = e.vector_mode (0);
+    if (BYTES_BIG_ENDIAN)
+      {
+	insn_code icode = code_for_aarch64_vec_duplicate_vq_be (mode);
+	unsigned int nunits = 128 / GET_MODE_UNIT_BITSIZE (mode);
+	rtx indices = aarch64_gen_stepped_int_parallel
+	  (nunits, nunits - 1, -1);
+
+	e.add_output_operand (icode);
+	e.add_input_operand (icode, e.args[0]);
+	e.add_fixed_operand (indices);
+	return e.generate_insn (icode);
+      }
+    insn_code icode = code_for_aarch64_vec_duplicate_vq_le (mode);
+    e.add_output_operand (icode);
+    e.add_input_operand (icode, e.args[0]);
+    return e.generate_insn (icode);
+  }
+};
+
 class svindex_impl : public function_base
 {
 public:
@@ -1305,7 +1412,7 @@ public:
       icode = convert_optab_handler (maskload_optab,
 				     e.vector_mode (0), e.gp_mode (0));
     else
-      icode = code_for_aarch64_ld1 (e.tuple_mode (0));
+      icode = code_for_aarch64 (UNSPEC_LD1_COUNT, e.tuple_mode (0));
     return e.use_contiguous_load_insn (icode);
   }
 };
@@ -1605,7 +1712,10 @@ public:
   rtx
   expand (function_expander &e) const override
   {
-    insn_code icode = code_for_aarch64_ldnt1 (e.tuple_mode (0));
+    insn_code icode = (e.vectors_per_tuple () == 1
+		       ? code_for_aarch64_ldnt1 (e.vector_mode (0))
+		       : code_for_aarch64 (UNSPEC_LDNT1_COUNT,
+					   e.tuple_mode (0)));
     return e.use_contiguous_load_insn (icode);
   }
 };
@@ -2415,7 +2525,7 @@ public:
       icode = convert_optab_handler (maskstore_optab,
 				     e.vector_mode (0), e.gp_mode (0));
     else
-      icode = code_for_aarch64_st1 (e.tuple_mode (0));
+      icode = code_for_aarch64 (UNSPEC_ST1_COUNT, e.tuple_mode (0));
     return e.use_contiguous_store_insn (icode);
   }
 };
@@ -2533,7 +2643,10 @@ public:
   rtx
   expand (function_expander &e) const override
   {
-    insn_code icode = code_for_aarch64_stnt1 (e.tuple_mode (0));
+    insn_code icode = (e.vectors_per_tuple () == 1
+		       ? code_for_aarch64_stnt1 (e.vector_mode (0))
+		       : code_for_aarch64 (UNSPEC_STNT1_COUNT,
+					   e.tuple_mode (0)));
     return e.use_contiguous_store_insn (icode);
   }
 };
@@ -3116,5 +3229,8 @@ FUNCTION (svzip1q, unspec_based_function, (UNSPEC_ZIP1Q, UNSPEC_ZIP1Q,
 FUNCTION (svzip2, svzip_impl, (1))
 FUNCTION (svzip2q, unspec_based_function, (UNSPEC_ZIP2Q, UNSPEC_ZIP2Q,
 					   UNSPEC_ZIP2Q))
+NEON_SVE_BRIDGE_FUNCTION (svget_neonq, svget_neonq_impl,)
+NEON_SVE_BRIDGE_FUNCTION (svset_neonq, svset_neonq_impl,)
+NEON_SVE_BRIDGE_FUNCTION (svdup_neonq, svdup_neonq_impl,)
 
 } /* end namespace aarch64_sve */

gcc/config/aarch64/aarch64-sve-builtins-base.h

@@ -299,6 +299,12 @@ namespace aarch64_sve
     extern const function_base *const svzip2;
     extern const function_base *const svzip2q;
   }
+  namespace neon_sve_bridge_functions
+  {
+    extern const function_base *const svset_neonq;
+    extern const function_base *const svget_neonq;
+    extern const function_base *const svdup_neonq;
+  }
 }
 
 #endif

gcc/config/aarch64/aarch64-sve-builtins-functions.h

@@ -840,4 +840,8 @@ public:
   namespace { static CONSTEXPR const CLASS NAME##_obj ARGS; } \
   namespace functions { const function_base *const NAME = &NAME##_obj; }
 
+#define NEON_SVE_BRIDGE_FUNCTION(NAME, CLASS, ARGS) \
+  namespace { static CONSTEXPR const CLASS NAME##_obj ARGS; } \
+  namespace neon_sve_bridge_functions { const function_base *const NAME = &NAME##_obj; }
+
 #endif

gcc/config/aarch64/aarch64-sve-builtins-shapes.cc

@@ -29,6 +29,7 @@
 #include "optabs.h"
 #include "aarch64-sve-builtins.h"
 #include "aarch64-sve-builtins-shapes.h"
+#include "aarch64-builtins.h"
 
 /* In the comments below, _t0 represents the first type suffix and _t1
    represents the second.  Square brackets enclose characters that are
@@ -178,6 +179,8 @@ parse_element_type (const function_instance &instance, const char *&format)
    s<elt>  - a scalar type with the given element suffix
    t<elt>  - a vector or tuple type with given element suffix [*1]
    v<elt>  - a vector with the given element suffix
+   D<elt>  - a 64 bit neon vector
+   Q<elt>  - a 128 bit neon vector
 
    where <elt> has the format described above parse_element_type
 
@@ -261,6 +264,20 @@ parse_type (const function_instance &instance, const char *&format)
       return acle_vector_types[0][type_suffixes[suffix].vector_type];
     }
 
+  if (ch == 'D')
+    {
+      type_suffix_index suffix = parse_element_type (instance, format);
+      int neon_index = type_suffixes[suffix].neon64_type;
+      return aarch64_simd_types[neon_index].itype;
+    }
+
+  if (ch == 'Q')
+    {
+      type_suffix_index suffix = parse_element_type (instance, format);
+      int neon_index = type_suffixes[suffix].neon128_type;
+      return aarch64_simd_types[neon_index].itype;
+    }
+
   gcc_unreachable ();
 }
 
@@ -2476,6 +2493,67 @@ struct get_def : public overloaded_base<0>
 };
 SHAPE (get)
 
+/* <t0>xN_t svfoo[_t0](sv<t0>_t).  */
+struct get_neonq_def : public overloaded_base<0>
+{
+  void
+  build (function_builder &b, const function_group_info &group) const override
+  {
+    b.add_overloaded_functions (group, MODE_none);
+    build_all (b, "Q0,v0", group, MODE_none);
+  }
+  tree
+  resolve (function_resolver &r) const override
+  {
+    return r.resolve_unary ();
+  }
+};
+SHAPE (get_neonq)
+
+/* sv<t0>_t svfoo[_t0](sv<t0>_t, <t0>xN_t).  */
+struct set_neonq_def : public overloaded_base<0>
+{
+  void
+  build (function_builder &b, const function_group_info &group) const override
+  {
+    b.add_overloaded_functions (group, MODE_none);
+    build_all (b, "v0,v0,Q0", group, MODE_none);
+  }
+  tree
+  resolve (function_resolver &r) const override
+  {
+    unsigned int i, nargs;
+    type_suffix_index type;
+    if (!r.check_gp_argument (2, i, nargs)
+	|| (type = r.infer_neon128_vector_type (i + 1)) == NUM_TYPE_SUFFIXES)
+      return error_mark_node;
+    return r.resolve_to (r.mode_suffix_id, type);
+  }
+};
+SHAPE (set_neonq)
+
+/* sv<t0>_t svfoo[_t0](<t0>xN_t).  */
+struct dup_neonq_def : public overloaded_base<0>
+{
+  void
+  build (function_builder &b, const function_group_info &group) const override
+  {
+    b.add_overloaded_functions (group, MODE_none);
+    build_all (b, "v0,Q0", group, MODE_none);
+  }
+  tree
+  resolve (function_resolver &r) const override
+  {
+    unsigned int i, nargs;
+    type_suffix_index type;
+    if (!r.check_gp_argument (1, i, nargs)
+	|| (type = r.infer_neon128_vector_type (i)) == NUM_TYPE_SUFFIXES)
+      return error_mark_node;
+    return r.resolve_to (r.mode_suffix_id, type);
+  }
+};
+SHAPE (dup_neonq)
+
 /* sv<t0>_t svfoo[_t0](sv<t0>_t, uint64_t)
    <t0>_t svfoo[_n_t0](<t0>_t, uint64_t)
 

gcc/config/aarch64/aarch64-sve-builtins-shapes.h

@@ -126,10 +126,12 @@ namespace aarch64_sve
     extern const function_shape *const dot_za_slice_lane;
     extern const function_shape *const dot_za_slice_uint_lane;
     extern const function_shape *const dupq;
+    extern const function_shape *const dup_neonq;
     extern const function_shape *const ext;
     extern const function_shape *const extract_pred;
     extern const function_shape *const fold_left;
     extern const function_shape *const get;
+    extern const function_shape *const get_neonq;
     extern const function_shape *const inc_dec;
     extern const function_shape *const inc_dec_pat;
     extern const function_shape *const inc_dec_pred;
@@ -170,6 +172,7 @@ namespace aarch64_sve
     extern const function_shape *const select_pred;
     extern const function_shape *const set;
     extern const function_shape *const setffr;
+    extern const function_shape *const set_neonq;
     extern const function_shape *const shift_left_imm_long;
     extern const function_shape *const shift_left_imm_to_uint;
     extern const function_shape *const shift_right_imm;

gcc/config/aarch64/aarch64-sve-builtins-sme.cc

@@ -365,7 +365,8 @@ public:
   expand (function_expander &e) const override
   {
     machine_mode mode = e.vectors_per_tuple () == 4 ? VNx8DImode : VNx4DImode;
-    return e.use_exact_insn (code_for_aarch64_sme_read (mode));
+    rtx res = e.use_exact_insn (code_for_aarch64_sme_read (mode));
+    return aarch64_sve_reinterpret (e.result_mode (), res);
   }
 };
 
@@ -457,7 +458,7 @@ public:
   expand (function_expander &e) const override
   {
     machine_mode mode = e.vectors_per_tuple () == 4 ? VNx8DImode : VNx4DImode;
-    e.args[1] = lowpart_subreg (mode, e.args[1], e.tuple_mode (1));
+    e.args[1] = aarch64_sve_reinterpret (mode, e.args[1]);
     return e.use_exact_insn (code_for_aarch64_sme_write (mode));
   }
 };

gcc/config/aarch64/aarch64-sve-builtins.cc

@@ -53,6 +53,7 @@
 #include "aarch64-sve-builtins-sve2.h"
 #include "aarch64-sve-builtins-sme.h"
 #include "aarch64-sve-builtins-shapes.h"
+#include "aarch64-builtins.h"
 
 namespace aarch64_sve {
 
@@ -129,7 +130,8 @@ CONSTEXPR const mode_suffix_info mode_suffixes[] = {
 
 /* Static information about each type_suffix_index.  */
 CONSTEXPR const type_suffix_info type_suffixes[NUM_TYPE_SUFFIXES + 1] = {
-#define DEF_SVE_TYPE_SUFFIX(NAME, ACLE_TYPE, CLASS, BITS, MODE) \
+#define DEF_SVE_NEON_TYPE_SUFFIX(NAME, ACLE_TYPE, CLASS, BITS, MODE, \
+				 NEON64, NEON128) \
   { "_" #NAME, \
     VECTOR_TYPE_##ACLE_TYPE, \
     TYPE_##CLASS, \
@@ -142,7 +144,12 @@ CONSTEXPR const type_suffix_info type_suffixes[NUM_TYPE_SUFFIXES + 1] = {
     TYPE_##CLASS == TYPE_bool, \
     false, \
     0, \
-    MODE },
+    MODE, \
+    NEON64, \
+    NEON128 },
+#define DEF_SVE_TYPE_SUFFIX(NAME, ACLE_TYPE, CLASS, BITS, MODE) \
+  DEF_SVE_NEON_TYPE_SUFFIX (NAME, ACLE_TYPE, CLASS, BITS, MODE, \
+			    ARM_NEON_H_TYPES_LAST, ARM_NEON_H_TYPES_LAST)
 #define DEF_SME_ZA_SUFFIX(NAME, BITS, MODE) \
   { "_" #NAME, \
     NUM_VECTOR_TYPES, \
@@ -156,10 +163,12 @@ CONSTEXPR const type_suffix_info type_suffixes[NUM_TYPE_SUFFIXES + 1] = {
     false, \
     true, \
     0, \
-    MODE },
+    MODE, \
+    ARM_NEON_H_TYPES_LAST, \
+    ARM_NEON_H_TYPES_LAST },
 #include "aarch64-sve-builtins.def"
   { "", NUM_VECTOR_TYPES, TYPE_bool, 0, 0, false, false, false, false,
-    false, false, 0, VOIDmode }
+    false, false, 0, VOIDmode, ARM_NEON_H_TYPES_LAST, ARM_NEON_H_TYPES_LAST }
 };
 
 CONSTEXPR const group_suffix_info group_suffixes[] = {
@@ -884,6 +893,14 @@ static CONSTEXPR const function_group_info function_groups[] = {
 #include "aarch64-sve-builtins.def"
 };
 
+/* A list of all NEON-SVE-Bridge ACLE functions.  */
+static CONSTEXPR const function_group_info neon_sve_function_groups[] = {
+#define DEF_NEON_SVE_FUNCTION(NAME, SHAPE, TYPES, GROUPS, PREDS) \
+  { #NAME, &neon_sve_bridge_functions::NAME, &shapes::SHAPE, types_##TYPES, \
+    groups_##GROUPS, preds_##PREDS, 0 },
+#include "aarch64-neon-sve-bridge-builtins.def"
+};
+
 /* The scalar type associated with each vector type.  */
 extern GTY(()) tree scalar_types[NUM_VECTOR_TYPES + 1];
 tree scalar_types[NUM_VECTOR_TYPES + 1];
@@ -2092,6 +2109,33 @@ function_resolver::infer_integer_vector_type (unsigned int argno)
   return type;
 }
 
+/* Require argument ARGNO to have some form of NEON128 vector type.  Return the
+   associated type suffix on success.
+   Report an error and return NUM_TYPE_SUFFIXES on failure.  */
+type_suffix_index
+function_resolver::infer_neon128_vector_type (unsigned int argno)
+{
+  tree actual = get_argument_type (argno);
+  if (actual == error_mark_node)
+    return NUM_TYPE_SUFFIXES;
+
+  for (unsigned int suffix_i = 0; suffix_i < NUM_TYPE_SUFFIXES; ++suffix_i)
+    {
+      int neon_index = type_suffixes[suffix_i].neon128_type;
+      if (neon_index != ARM_NEON_H_TYPES_LAST)
+	{
+	  tree type = aarch64_simd_types[neon_index].itype;
+	  if (type && matches_type_p (type, actual))
+	    return type_suffix_index (suffix_i);
+	}
+    }
+
+  error_at (location, "passing %qT to argument %d of %qE, which"
+	    " expects a 128 bit NEON vector type", actual, argno + 1, fndecl);
+  return NUM_TYPE_SUFFIXES;
+}
+
+
 /* Like infer_vector_type, but also require the type to be an unsigned
    integer.  */
 type_suffix_index
@@ -4454,6 +4498,7 @@ init_builtins ()
     {
       handle_arm_sve_h ();
       handle_arm_sme_h ();
+      handle_arm_neon_sve_bridge_h ();
     }
 }
 
@@ -4588,6 +4633,16 @@ handle_arm_sve_h ()
       builder.register_function_group (function_groups[i]);
 }
 
+/* Implement #pragma GCC aarch64 "arm_neon_sve_bridge.h".  */
+void
+handle_arm_neon_sve_bridge_h ()
+{
+  /* Define the functions.  */
+  function_builder builder;
+  for (unsigned int i = 0; i < ARRAY_SIZE (neon_sve_function_groups); ++i)
+    builder.register_function_group (neon_sve_function_groups[i]);
+}
+
 /* Return the function decl with SVE function subcode CODE, or error_mark_node
    if no such function exists.  */
 tree

gcc/config/aarch64/aarch64-sve-builtins.def

@@ -41,6 +41,11 @@
 #define DEF_SVE_FUNCTION_GS(A, B, C, D, E)
 #endif
 
+#ifndef DEF_SVE_NEON_TYPE_SUFFIX
+#define DEF_SVE_NEON_TYPE_SUFFIX(A, B, C, D, E, F, G) \
+  DEF_SVE_TYPE_SUFFIX(A, B, C, D, E)
+#endif
+
 #ifndef DEF_SVE_FUNCTION
 #define DEF_SVE_FUNCTION(NAME, SHAPE, TYPES, PREDS) \
   DEF_SVE_FUNCTION_GS (NAME, SHAPE, TYPES, none, PREDS)
@@ -107,23 +112,35 @@ DEF_SVE_TYPE_SUFFIX (b8, svbool_t, bool, 8, VNx16BImode)
 DEF_SVE_TYPE_SUFFIX (b16, svbool_t, bool, 16, VNx8BImode)
 DEF_SVE_TYPE_SUFFIX (b32, svbool_t, bool, 32, VNx4BImode)
 DEF_SVE_TYPE_SUFFIX (b64, svbool_t, bool, 64, VNx2BImode)
-DEF_SVE_TYPE_SUFFIX (bf16, svbfloat16_t, bfloat, 16, VNx8BFmode)
 DEF_SVE_TYPE_SUFFIX (c, svcount_t, count, 8, VNx16BImode)
 DEF_SVE_TYPE_SUFFIX (c8, svcount_t, count, 8, VNx16BImode)
 DEF_SVE_TYPE_SUFFIX (c16, svcount_t, count, 16, VNx16BImode)
 DEF_SVE_TYPE_SUFFIX (c32, svcount_t, count, 32, VNx16BImode)
 DEF_SVE_TYPE_SUFFIX (c64, svcount_t, count, 64, VNx16BImode)
-DEF_SVE_TYPE_SUFFIX (f16, svfloat16_t, float, 16, VNx8HFmode)
-DEF_SVE_TYPE_SUFFIX (f32, svfloat32_t, float, 32, VNx4SFmode)
-DEF_SVE_TYPE_SUFFIX (f64, svfloat64_t, float, 64, VNx2DFmode)
-DEF_SVE_TYPE_SUFFIX (s8, svint8_t, signed, 8, VNx16QImode)
-DEF_SVE_TYPE_SUFFIX (s16, svint16_t, signed, 16, VNx8HImode)
-DEF_SVE_TYPE_SUFFIX (s32, svint32_t, signed, 32, VNx4SImode)
-DEF_SVE_TYPE_SUFFIX (s64, svint64_t, signed, 64, VNx2DImode)
-DEF_SVE_TYPE_SUFFIX (u8, svuint8_t, unsigned, 8, VNx16QImode)
-DEF_SVE_TYPE_SUFFIX (u16, svuint16_t, unsigned, 16, VNx8HImode)
-DEF_SVE_TYPE_SUFFIX (u32, svuint32_t, unsigned, 32, VNx4SImode)
-DEF_SVE_TYPE_SUFFIX (u64, svuint64_t, unsigned, 64, VNx2DImode)
+DEF_SVE_NEON_TYPE_SUFFIX (bf16, svbfloat16_t, bfloat, 16, VNx8BFmode,
+			  Bfloat16x4_t, Bfloat16x8_t)
+DEF_SVE_NEON_TYPE_SUFFIX (f16, svfloat16_t, float, 16, VNx8HFmode,
+			  Float16x4_t, Float16x8_t)
+DEF_SVE_NEON_TYPE_SUFFIX (f32, svfloat32_t, float, 32, VNx4SFmode,
+			  Float32x2_t, Float32x4_t)
+DEF_SVE_NEON_TYPE_SUFFIX (f64, svfloat64_t, float, 64, VNx2DFmode,
+			  Float64x1_t, Float64x2_t)
+DEF_SVE_NEON_TYPE_SUFFIX (s8, svint8_t, signed, 8, VNx16QImode,
+			  Int8x8_t, Int8x16_t)
+DEF_SVE_NEON_TYPE_SUFFIX (s16, svint16_t, signed, 16, VNx8HImode,
+			  Int16x4_t, Int16x8_t)
+DEF_SVE_NEON_TYPE_SUFFIX (s32, svint32_t, signed, 32, VNx4SImode,
+			  Int32x2_t, Int32x4_t)
+DEF_SVE_NEON_TYPE_SUFFIX (s64, svint64_t, signed, 64, VNx2DImode,
+			  Int64x1_t, Int64x2_t)
+DEF_SVE_NEON_TYPE_SUFFIX (u8, svuint8_t, unsigned, 8, VNx16QImode,
+			  Uint8x8_t, Uint8x16_t)
+DEF_SVE_NEON_TYPE_SUFFIX (u16, svuint16_t, unsigned, 16, VNx8HImode,
+			  Uint16x4_t, Uint16x8_t)
+DEF_SVE_NEON_TYPE_SUFFIX (u32, svuint32_t, unsigned, 32, VNx4SImode,
+			  Uint32x2_t, Uint32x4_t)
+DEF_SVE_NEON_TYPE_SUFFIX (u64, svuint64_t, unsigned, 64, VNx2DImode,
+			  Uint64x1_t, Uint64x2_t)
 
 /* Associate _za with bytes.  This is needed for svldr_vnum_za and
    svstr_vnum_za, whose ZA offset can be in the range [0, 15], as for za8.  */
@@ -159,6 +176,7 @@ DEF_SVE_GROUP_SUFFIX (vg4x4, 4, 4)
 #undef DEF_SVE_FUNCTION_GS
 #undef DEF_SVE_GROUP_SUFFIX
 #undef DEF_SME_ZA_SUFFIX
+#undef DEF_SVE_NEON_TYPE_SUFFIX
 #undef DEF_SVE_TYPE_SUFFIX
 #undef DEF_SVE_TYPE
 #undef DEF_SVE_MODE

gcc/config/aarch64/aarch64-sve-builtins.h

@@ -20,6 +20,8 @@
 #ifndef GCC_AARCH64_SVE_BUILTINS_H
 #define GCC_AARCH64_SVE_BUILTINS_H
 
+#include "aarch64-builtins.h"
+
 /* The full name of an SVE ACLE function is the concatenation of:
 
    - the base name ("svadd", etc.)
@@ -229,6 +231,14 @@ struct mode_suffix_info
   units_index displacement_units;
 };
 
+#define ENTRY(E, M, Q, G) E,
+enum aarch64_simd_type
+{
+#include "aarch64-simd-builtin-types.def"
+  ARM_NEON_H_TYPES_LAST
+};
+#undef ENTRY
+
 /* Static information about a type suffix.  */
 struct type_suffix_info
 {
@@ -262,6 +272,11 @@ struct type_suffix_info
 
   /* The associated vector or predicate mode.  */
   machine_mode vector_mode : 16;
+
+  /* The corresponding 64-bit and 128-bit arm_neon.h types, or
+     ARM_NEON_H_TYPES_LAST if none.  */
+  aarch64_simd_type neon64_type;
+  aarch64_simd_type neon128_type;
 };
 
 /* Static information about a group suffix.  */
@@ -498,6 +513,7 @@ public:
   sve_type infer_vector_or_tuple_type (unsigned int, unsigned int);
   type_suffix_index infer_vector_type (unsigned int);
   type_suffix_index infer_integer_vector_type (unsigned int);
+  type_suffix_index infer_neon128_vector_type (unsigned int);
   type_suffix_index infer_unsigned_vector_type (unsigned int);
   type_suffix_index infer_sd_vector_type (unsigned int);
   sve_type infer_tuple_type (unsigned int);

gcc/config/aarch64/aarch64-sve.md

@@ -1277,17 +1277,6 @@
   "ld1<Vesize>\t%0.<Vctype>, %2/z, %1"
 )
 
-;; Predicated LD1 (multi), with a count as predicate.
-(define_insn "@aarch64_ld1<mode>"
-  [(set (match_operand:SVE_FULLx24 0 "aligned_register_operand" "=Uw<vector_count>")
-	(unspec:SVE_FULLx24
-	  [(match_operand:VNx16BI 2 "register_operand" "Uph")
-	   (match_operand:SVE_FULLx24 1 "memory_operand" "m")]
-	  UNSPEC_LD1_SVE_COUNT))]
-  "TARGET_SME2 && TARGET_STREAMING"
-  "ld1<Vesize>\t%0, %K2/z, %1"
-)
-
 ;; Unpredicated LD[234].
 (define_expand "vec_load_lanes<mode><vsingle>"
   [(set (match_operand:SVE_STRUCT 0 "register_operand")
@@ -1430,17 +1419,6 @@
   "ldnt1<Vesize>\t%0.<Vetype>, %2/z, %1"
 )
 
-;; Predicated contiguous non-temporal load (multi).
-(define_insn "@aarch64_ldnt1<mode>"
-  [(set (match_operand:SVE_FULLx24 0 "aligned_register_operand" "=Uw<vector_count>")
-	(unspec:SVE_FULLx24
-	  [(match_operand:VNx16BI 2 "register_operand" "Uph")
-	   (match_operand:SVE_FULLx24 1 "memory_operand" "m")]
-	  UNSPEC_LDNT1_SVE_COUNT))]
-  "TARGET_SVE"
-  "ldnt1<Vesize>\t%0, %K2/z, %1"
-)
-
 ;; -------------------------------------------------------------------------
 ;; ---- Normal gather loads
 ;; -------------------------------------------------------------------------
@@ -2263,17 +2241,6 @@
   "st1<Vesize>\t%1.<Vctype>, %2, %0"
 )
 
-(define_insn "@aarch64_st1<mode>"
-  [(set (match_operand:SVE_FULLx24 0 "memory_operand" "+m")
-	(unspec:SVE_FULLx24
-	  [(match_operand:VNx16BI 2 "register_operand" "Uph")
-	   (match_operand:SVE_FULLx24 1 "aligned_register_operand" "Uw<vector_count>")
-	   (match_dup 0)]
-	  UNSPEC_ST1_SVE_COUNT))]
-  "TARGET_SME2 && TARGET_STREAMING"
-  "st1<Vesize>\t%1, %K2, %0"
-)
-
 ;; Unpredicated ST[234].  This is always a full update, so the dependence
 ;; on the old value of the memory location (via (match_dup 0)) is redundant.
 ;; There doesn't seem to be any obvious benefit to treating the all-true
@@ -2373,17 +2340,6 @@
   "stnt1<Vesize>\t%1.<Vetype>, %2, %0"
 )
 
-(define_insn "@aarch64_stnt1<mode>"
-  [(set (match_operand:SVE_FULLx24 0 "memory_operand" "+m")
-	(unspec:SVE_FULLx24
-	  [(match_operand:VNx16BI 2 "register_operand" "Uph")
-	   (match_operand:SVE_FULLx24 1 "aligned_register_operand" "Uw<vector_count>")
-	   (match_dup 0)]
-	  UNSPEC_STNT1_SVE_COUNT))]
-  "TARGET_SME2 && TARGET_STREAMING"
-  "stnt1<Vesize>\t%1, %K2, %0"
-)
-
 ;; -------------------------------------------------------------------------
 ;; ---- Normal scatter stores
 ;; -------------------------------------------------------------------------
@@ -10994,3 +10950,36 @@
     operands[4] = CONSTM1_RTX (<VPRED>mode);
   }
 )
+
+(define_insn_and_split "@aarch64_sve_get_neonq_<mode>"
+  [(set (match_operand:<V128> 0 "register_operand" "=w")
+	  (vec_select:<V128>
+	    (match_operand:SVE_FULL 1 "register_operand" "w")
+	    (match_operand 2 "descending_int_parallel")))]
+  "TARGET_SVE
+   && BYTES_BIG_ENDIAN
+   && known_eq (INTVAL (XVECEXP (operands[2], 0, 0)),
+		GET_MODE_NUNITS (<V128>mode) - 1)"
+  "#"
+  "&& reload_completed"
+  [(set (match_dup 0) (match_dup 1))]
+  {
+    operands[1] = gen_rtx_REG (<V128>mode, REGNO (operands[1]));
+  }
+)
+
+(define_insn "@aarch64_sve_set_neonq_<mode>"
+  [(set (match_operand:SVE_FULL 0 "register_operand" "=w")
+      (unspec:SVE_FULL
+	[(match_operand:SVE_FULL 1 "register_operand" "w")
+	(match_operand:<V128> 2 "register_operand" "w")
+	(match_operand:<VPRED> 3 "register_operand" "Upl")]
+	UNSPEC_SET_NEONQ))]
+  "TARGET_SVE
+   && BYTES_BIG_ENDIAN"
+  {
+    operands[2] = lowpart_subreg (<MODE>mode, operands[2],
+                                  GET_MODE (operands[2]));
+    return "sel\t%0.<Vetype>, %3, %2.<Vetype>, %1.<Vetype>";
+  }
+)

gcc/config/aarch64/aarch64-sve2.md

@@ -21,8 +21,12 @@
 ;; The file is organised into the following sections (search for the full
 ;; line):
 ;;
-;; == Moves
+;; == Loads
+;; ---- Multi-register loads predicated by a counter
 ;; ---- Non-temporal gather loads
+;;
+;; == Stores
+;; ---- Multi-register stores predicated by a counter
 ;; ---- Non-temporal scatter stores
 ;;
 ;; == Predicate manipulation
@@ -112,9 +116,85 @@
 ;; ---- Optional SM4 extensions
 
 ;; =========================================================================
-;; == Moves
+;; == Loads
 ;; =========================================================================
 
+;; -------------------------------------------------------------------------
+;; ---- Multi-register loads predicated by a counter
+;; -------------------------------------------------------------------------
+;; Includes:
+;; - LD1B
+;; - LD1D
+;; - LD1H
+;; - LD1W
+;; - LDNT1B
+;; - LDNT1D
+;; - LDNT1H
+;; - LDNT1W
+;; -------------------------------------------------------------------------
+
+;; Predicated LD1 (multi), with a count as predicate.
+(define_insn "@aarch64_<optab><mode>"
+  [(set (match_operand:SVE_FULLx24 0 "aligned_register_operand" "=Uw<vector_count>")
+	(unspec:SVE_FULLx24
+	  [(match_operand:VNx16BI 2 "register_operand" "Uph")
+	   (match_operand:SVE_FULLx24 1 "memory_operand" "m")]
+	  LD1_COUNT))]
+  "TARGET_STREAMING_SME2"
+  "<optab><Vesize>\t%0, %K2/z, %1"
+  [(set_attr "stride_type" "ld1_consecutive")]
+)
+
+(define_insn "@aarch64_<optab><mode>_strided2"
+  [(set (match_operand:<VSINGLE> 0 "aarch64_simd_register" "=Uwd")
+	(unspec:<VSINGLE>
+	  [(match_operand:VNx16BI 3 "register_operand" "Uph")
+	   (match_operand:SVE_FULLx2 2 "memory_operand" "m")
+	   (const_int 0)]
+	  LD1_COUNT))
+   (set (match_operand:<VSINGLE> 1 "aarch64_simd_register" "=w")
+	(unspec:<VSINGLE>
+	  [(match_dup 3)
+	   (match_dup 2)
+	   (const_int 1)]
+	  LD1_COUNT))]
+  "TARGET_STREAMING_SME2
+   && aarch64_strided_registers_p (operands, 2, 8)"
+  "<optab><Vesize>\t{%0.<Vetype>, %1.<Vetype>}, %K3/z, %2"
+  [(set_attr "stride_type" "ld1_strided")]
+)
+
+(define_insn "@aarch64_<optab><mode>_strided4"
+  [(set (match_operand:<VSINGLE> 0 "aarch64_simd_register" "=Uwt")
+	(unspec:<VSINGLE>
+	  [(match_operand:VNx16BI 5 "register_operand" "Uph")
+	   (match_operand:SVE_FULLx4 4 "memory_operand" "m")
+	   (const_int 0)]
+	  LD1_COUNT))
+   (set (match_operand:<VSINGLE> 1 "aarch64_simd_register" "=w")
+	(unspec:<VSINGLE>
+	  [(match_dup 5)
+	   (match_dup 4)
+	   (const_int 1)]
+	  LD1_COUNT))
+   (set (match_operand:<VSINGLE> 2 "aarch64_simd_register" "=w")
+	(unspec:<VSINGLE>
+	  [(match_dup 5)
+	   (match_dup 4)
+	   (const_int 2)]
+	  LD1_COUNT))
+   (set (match_operand:<VSINGLE> 3 "aarch64_simd_register" "=w")
+	(unspec:<VSINGLE>
+	  [(match_dup 5)
+	   (match_dup 4)
+	   (const_int 3)]
+	  LD1_COUNT))]
+  "TARGET_STREAMING_SME2
+   && aarch64_strided_registers_p (operands, 4, 4)"
+  "<optab><Vesize>\t{%0.<Vetype>, %1.<Vetype>, %2.<Vetype>, %3.<Vetype>}, %K5/z, %4"
+  [(set_attr "stride_type" "ld1_strided")]
+)
+
 ;; -------------------------------------------------------------------------
 ;; ---- Non-temporal gather loads
 ;; -------------------------------------------------------------------------
@@ -171,6 +251,66 @@
   }
 )
 
+;; =========================================================================
+;; == Stores
+;; =========================================================================
+
+;; -------------------------------------------------------------------------
+;; ---- Multi-register stores predicated by a counter
+;; -------------------------------------------------------------------------
+;; Includes:
+;; - ST1B
+;; - ST1D
+;; - ST1H
+;; - ST1W
+;; - STNT1B
+;; - STNT1D
+;; - STNT1H
+;; - STNT1W
+;; -------------------------------------------------------------------------
+
+(define_insn "@aarch64_<optab><mode>"
+  [(set (match_operand:SVE_FULLx24 0 "memory_operand" "+m")
+	(unspec:SVE_FULLx24
+	  [(match_operand:VNx16BI 2 "register_operand" "Uph")
+	   (match_operand:SVE_FULLx24 1 "aligned_register_operand" "Uw<vector_count>")
+	   (match_dup 0)]
+	  ST1_COUNT))]
+  "TARGET_STREAMING_SME2"
+  "<optab><Vesize>\t%1, %K2, %0"
+  [(set_attr "stride_type" "st1_consecutive")]
+)
+
+(define_insn "@aarch64_<optab><mode>_strided2"
+  [(set (match_operand:SVE_FULLx24 0 "memory_operand" "+m")
+	(unspec:SVE_FULLx24
+	  [(match_operand:VNx16BI 1 "register_operand" "Uph")
+	   (match_operand:<VSINGLE> 2 "aarch64_simd_register" "Uwd")
+	   (match_operand:<VSINGLE> 3 "aarch64_simd_register" "w")
+	   (match_dup 0)]
+	  ST1_COUNT))]
+  "TARGET_STREAMING_SME2
+   && aarch64_strided_registers_p (operands + 2, 2, 8)"
+  "<optab><Vesize>\t{%2.<Vetype>, %3.<Vetype>}, %K1, %0"
+  [(set_attr "stride_type" "st1_strided")]
+)
+
+(define_insn "@aarch64_<optab><mode>_strided4"
+  [(set (match_operand:SVE_FULLx24 0 "memory_operand" "+m")
+	(unspec:SVE_FULLx24
+	  [(match_operand:VNx16BI 1 "register_operand" "Uph")
+	   (match_operand:<VSINGLE> 2 "aarch64_simd_register" "Uwt")
+	   (match_operand:<VSINGLE> 3 "aarch64_simd_register" "w")
+	   (match_operand:<VSINGLE> 4 "aarch64_simd_register" "w")
+	   (match_operand:<VSINGLE> 5 "aarch64_simd_register" "w")
+	   (match_dup 0)]
+	  ST1_COUNT))]
+  "TARGET_STREAMING_SME2
+   && aarch64_strided_registers_p (operands + 2, 4, 4)"
+  "<optab><Vesize>\t{%2.<Vetype>, %3.<Vetype>, %4.<Vetype>, %5.<Vetype>}, %K1, %0"
+  [(set_attr "stride_type" "st1_strided")]
+)
+
 ;; -------------------------------------------------------------------------
 ;; ---- Non-temporal scatter stores
 ;; -------------------------------------------------------------------------

gcc/config/aarch64/aarch64-sys-regs.def

@@ -419,6 +419,16 @@
   SYSREG ("fpcr",		CPENC (3,3,4,4,0),	0,			AARCH64_NO_FEATURES)
   SYSREG ("fpexc32_el2",	CPENC (3,4,5,3,0),	0,			AARCH64_NO_FEATURES)
   SYSREG ("fpsr",		CPENC (3,3,4,4,1),	0,			AARCH64_NO_FEATURES)
+  SYSREG ("gcspr_el0",		CPENC (3,3,2,5,1),    	F_ARCHEXT,		AARCH64_FEATURE (GCS))
+  SYSREG ("gcspr_el1",		CPENC (3,0,2,5,1),    	F_ARCHEXT,		AARCH64_FEATURE (GCS))
+  SYSREG ("gcspr_el2",		CPENC (3,4,2,5,1),    	F_ARCHEXT,		AARCH64_FEATURE (GCS))
+  SYSREG ("gcspr_el12",		CPENC (3,5,2,5,1),    	F_ARCHEXT,		AARCH64_FEATURE (GCS))
+  SYSREG ("gcspr_el3",		CPENC (3,6,2,5,1),    	F_ARCHEXT,		AARCH64_FEATURE (GCS))
+  SYSREG ("gcscre0_el1",	CPENC (3,0,2,5,2),    	F_ARCHEXT,		AARCH64_FEATURE (GCS))
+  SYSREG ("gcscr_el1",		CPENC (3,0,2,5,0),    	F_ARCHEXT,		AARCH64_FEATURE (GCS))
+  SYSREG ("gcscr_el2",		CPENC (3,4,2,5,0),    	F_ARCHEXT,		AARCH64_FEATURE (GCS))
+  SYSREG ("gcscr_el12",		CPENC (3,5,2,5,0),    	F_ARCHEXT,		AARCH64_FEATURE (GCS))
+  SYSREG ("gcscr_el3",		CPENC (3,6,2,5,0),    	F_ARCHEXT,		AARCH64_FEATURE (GCS))
   SYSREG ("gcr_el1",		CPENC (3,0,1,0,6),	F_ARCHEXT,		AARCH64_FEATURE (MEMTAG))
   SYSREG ("gmid_el1",		CPENC (3,1,0,0,4),	F_REG_READ|F_ARCHEXT,	AARCH64_FEATURE (MEMTAG))
   SYSREG ("gpccr_el3",		CPENC (3,6,2,1,6),	0,			AARCH64_NO_FEATURES)
@@ -584,7 +594,7 @@
   SYSREG ("oslar_el1",		CPENC (2,0,1,0,4),	F_REG_WRITE,		AARCH64_NO_FEATURES)
   SYSREG ("oslsr_el1",		CPENC (2,0,1,1,4),	F_REG_READ,		AARCH64_NO_FEATURES)
   SYSREG ("pan",		CPENC (3,0,4,2,3),	F_ARCHEXT,		AARCH64_FEATURE (PAN))
-  SYSREG ("par_el1",		CPENC (3,0,7,4,0),	0,			AARCH64_NO_FEATURES)
+  SYSREG ("par_el1",		CPENC (3,0,7,4,0),	F_REG_128,		AARCH64_NO_FEATURES)
   SYSREG ("pmbidr_el1",		CPENC (3,0,9,10,7),	F_REG_READ|F_ARCHEXT,	AARCH64_FEATURE (PROFILE))
   SYSREG ("pmblimitr_el1",	CPENC (3,0,9,10,0),	F_ARCHEXT,		AARCH64_FEATURE (PROFILE))
   SYSREG ("pmbptr_el1",		CPENC (3,0,9,10,1),	F_ARCHEXT,		AARCH64_FEATURE (PROFILE))
@@ -746,6 +756,8 @@
   SYSREG ("prlar_el2",		CPENC (3,4,6,8,1),	F_ARCHEXT,		AARCH64_FEATURE (V8R))
   SYSREG ("prselr_el1",		CPENC (3,0,6,2,1),	F_ARCHEXT,		AARCH64_FEATURE (V8R))
   SYSREG ("prselr_el2",		CPENC (3,4,6,2,1),	F_ARCHEXT,		AARCH64_FEATURE (V8R))
+  SYSREG ("rcwmask_el1",	CPENC (3,0,13,0,6),	F_ARCHEXT|F_REG_128,	AARCH64_FEATURE (THE))
+  SYSREG ("rcwsmask_el1",	CPENC (3,0,13,0,3),	F_ARCHEXT|F_REG_128,	AARCH64_FEATURE (THE))
   SYSREG ("revidr_el1",		CPENC (3,0,0,0,6),	F_REG_READ,		AARCH64_NO_FEATURES)
   SYSREG ("rgsr_el1",		CPENC (3,0,1,0,5),	F_ARCHEXT,		AARCH64_FEATURE (MEMTAG))
   SYSREG ("rmr_el1",		CPENC (3,0,12,0,2),	0,			AARCH64_NO_FEATURES)
@@ -1034,13 +1046,13 @@
   SYSREG ("trfcr_el1",		CPENC (3,0,1,2,1),	F_ARCHEXT,		AARCH64_FEATURE (V8_4A))
   SYSREG ("trfcr_el12",		CPENC (3,5,1,2,1),	F_ARCHEXT,		AARCH64_FEATURE (V8_4A))
   SYSREG ("trfcr_el2",		CPENC (3,4,1,2,1),	F_ARCHEXT,		AARCH64_FEATURE (V8_4A))
-  SYSREG ("ttbr0_el1",		CPENC (3,0,2,0,0),	0,			AARCH64_NO_FEATURES)
-  SYSREG ("ttbr0_el12",		CPENC (3,5,2,0,0),	F_ARCHEXT,		AARCH64_FEATURE (V8_1A))
-  SYSREG ("ttbr0_el2",		CPENC (3,4,2,0,0),	F_ARCHEXT,		AARCH64_FEATURE (V8A))
+  SYSREG ("ttbr0_el1",		CPENC (3,0,2,0,0),	F_REG_128,		AARCH64_NO_FEATURES)
+  SYSREG ("ttbr0_el12",		CPENC (3,5,2,0,0),	F_ARCHEXT|F_REG_128,	AARCH64_FEATURE (V8_1A))
+  SYSREG ("ttbr0_el2",		CPENC (3,4,2,0,0),	F_ARCHEXT|F_REG_128,	AARCH64_FEATURE (V8A))
   SYSREG ("ttbr0_el3",		CPENC (3,6,2,0,0),	0,			AARCH64_NO_FEATURES)
-  SYSREG ("ttbr1_el1",		CPENC (3,0,2,0,1),	0,			AARCH64_NO_FEATURES)
-  SYSREG ("ttbr1_el12",		CPENC (3,5,2,0,1),	F_ARCHEXT,		AARCH64_FEATURE (V8_1A))
-  SYSREG ("ttbr1_el2",		CPENC (3,4,2,0,1),	F_ARCHEXT,		AARCH64_FEATURES (2, V8A, V8_1A))
+  SYSREG ("ttbr1_el1",		CPENC (3,0,2,0,1),	F_REG_128,		AARCH64_NO_FEATURES)
+  SYSREG ("ttbr1_el12",		CPENC (3,5,2,0,1),	F_ARCHEXT|F_REG_128,	AARCH64_FEATURE (V8_1A))
+  SYSREG ("ttbr1_el2",		CPENC (3,4,2,0,1),	F_ARCHEXT|F_REG_128,	AARCH64_FEATURES (2, V8A, V8_1A))
   SYSREG ("uao",		CPENC (3,0,4,2,4),	F_ARCHEXT,		AARCH64_FEATURE (V8_2A))
   SYSREG ("vbar_el1",		CPENC (3,0,12,0,0),	0,			AARCH64_NO_FEATURES)
   SYSREG ("vbar_el12",		CPENC (3,5,12,0,0),	F_ARCHEXT,		AARCH64_FEATURE (V8_1A))
@@ -1057,8 +1069,8 @@
   SYSREG ("vstcr_el2",		CPENC (3,4,2,6,2),	F_ARCHEXT,		AARCH64_FEATURE (V8_4A))
   SYSREG ("vsttbr_el2",		CPENC (3,4,2,6,0),	F_ARCHEXT,		AARCH64_FEATURES (2, V8A, V8_4A))
   SYSREG ("vtcr_el2",		CPENC (3,4,2,1,2),	0,			AARCH64_NO_FEATURES)
-  SYSREG ("vttbr_el2",		CPENC (3,4,2,1,0),	F_ARCHEXT,		AARCH64_FEATURE (V8A))
+  SYSREG ("vttbr_el2",		CPENC (3,4,2,1,0),	F_ARCHEXT|F_REG_128,	AARCH64_FEATURE (V8A))
   SYSREG ("zcr_el1",		CPENC (3,0,1,2,0),	F_ARCHEXT,		AARCH64_FEATURE (SVE))
   SYSREG ("zcr_el12",		CPENC (3,5,1,2,0),	F_ARCHEXT,		AARCH64_FEATURE (SVE))
   SYSREG ("zcr_el2",		CPENC (3,4,1,2,0),	F_ARCHEXT,		AARCH64_FEATURE (SVE))
-  SYSREG ("zcr_el3",		CPENC (3,6,1,2,0),	F_ARCHEXT,		AARCH64_FEATURE (SVE))
+  SYSREG ("zcr_el3",		CPENC (3,6,1,2,0),	F_ARCHEXT,		AARCH64_FEATURE (SVE))

gcc/config/aarch64/aarch64.cc

@@ -351,8 +351,6 @@ bool aarch64_pcrelative_literal_loads;
 /* Global flag for whether frame pointer is enabled.  */
 bool aarch64_use_frame_pointer;
 
-char *accepted_branch_protection_string = NULL;
-
 /* Support for command line parsing of boolean flags in the tuning
    structures.  */
 struct aarch64_flag_desc
@@ -498,6 +496,8 @@ typedef struct {
 #define F_ARCHEXT		   (1 << 4)
 /* Flag indicating register name is alias for another system register.  */
 #define F_REG_ALIAS		   (1 << 5)
+/* Flag indicatinig registers which may be implemented with 128-bits.  */
+#define F_REG_128		   (1 << 6)
 
 /* Database of system registers, their encodings and architectural
    requirements.  */
@@ -3224,6 +3224,28 @@ aarch64_split_simd_move (rtx dst, rtx src)
     }
 }
 
+/* Return a register that contains SVE value X reinterpreted as SVE mode MODE.
+   The semantics of those of svreinterpret rather than those of subregs;
+   see the comment at the head of aarch64-sve.md for details about the
+   difference.  */
+
+rtx
+aarch64_sve_reinterpret (machine_mode mode, rtx x)
+{
+  if (GET_MODE (x) == mode)
+    return x;
+
+  /* can_change_mode_class must only return true if subregs and svreinterprets
+     have the same semantics.  */
+  if (targetm.can_change_mode_class (GET_MODE (x), mode, FP_REGS))
+    return lowpart_subreg (mode, x, GET_MODE (x));
+
+  rtx res = gen_reg_rtx (mode);
+  x = force_reg (GET_MODE (x), x);
+  emit_insn (gen_aarch64_sve_reinterpret (mode, res, x));
+  return res;
+}
+
 bool
 aarch64_zero_extend_const_eq (machine_mode xmode, rtx x,
 			      machine_mode ymode, rtx y)
@@ -4954,14 +4976,17 @@ aarch64_sme_mode_switch_regs::add_reg (machine_mode mode, unsigned int regno)
   gcc_assert ((vec_flags & VEC_STRUCT) || end_regno == regno + 1);
   for (; regno < end_regno; regno++)
     {
+      /* Force the mode of SVE saves and restores even for single registers.
+	 This is necessary because big-endian targets only allow LDR Z and
+	 STR Z to be used with byte modes.  */
       machine_mode submode = mode;
-      if (vec_flags & VEC_STRUCT)
+      if (vec_flags & VEC_SVE_PRED)
+	submode = VNx16BImode;
+      else if (vec_flags & VEC_SVE_DATA)
+	submode = SVE_BYTE_MODE;
+      else if (vec_flags & VEC_STRUCT)
 	{
-	  if (vec_flags & VEC_SVE_PRED)
-	    submode = VNx16BImode;
-	  else if (vec_flags & VEC_SVE_DATA)
-	    submode = SVE_BYTE_MODE;
-	  else if (vec_flags & VEC_PARTIAL)
+	  if (vec_flags & VEC_PARTIAL)
 	    submode = V8QImode;
 	  else
 	    submode = V16QImode;
@@ -17972,12 +17997,6 @@ aarch64_adjust_generic_arch_tuning (struct tune_params &current_tune)
 static void
 aarch64_override_options_after_change_1 (struct gcc_options *opts)
 {
-  if (accepted_branch_protection_string)
-    {
-      opts->x_aarch64_branch_protection_string
-	= xstrdup (accepted_branch_protection_string);
-    }
-
   /* PR 70044: We have to be careful about being called multiple times for the
      same function.  This means all changes should be repeatable.  */
 
@@ -18544,7 +18563,8 @@ aarch64_override_options (void)
     aarch64_validate_sls_mitigation (aarch64_harden_sls_string);
 
   if (aarch64_branch_protection_string)
-    aarch_validate_mbranch_protection (aarch64_branch_protection_string);
+    aarch_validate_mbranch_protection (aarch64_branch_protection_string,
+				       "-mbranch-protection=");
 
   /* -mcpu=CPU is shorthand for -march=ARCH_FOR_CPU, -mtune=CPU.
      If either of -march or -mtune is given, they override their
@@ -18620,7 +18640,7 @@ aarch64_override_options (void)
   /* Return address signing is currently not supported for ILP32 targets.  For
      LP64 targets use the configured option in the absence of a command-line
      option for -mbranch-protection.  */
-  if (!TARGET_ILP32 && accepted_branch_protection_string == NULL)
+  if (!TARGET_ILP32 && aarch64_branch_protection_string == NULL)
     {
 #ifdef TARGET_ENABLE_PAC_RET
       aarch_ra_sign_scope = AARCH_FUNCTION_NON_LEAF;
@@ -18978,34 +18998,12 @@ aarch64_handle_attr_cpu (const char *str)
 
 /* Handle the argument STR to the branch-protection= attribute.  */
 
- static bool
- aarch64_handle_attr_branch_protection (const char* str)
- {
-  char *err_str = (char *) xmalloc (strlen (str) + 1);
-  enum aarch_parse_opt_result res = aarch_parse_branch_protection (str,
-								   &err_str);
-  bool success = false;
-  switch (res)
-    {
-     case AARCH_PARSE_MISSING_ARG:
-       error ("missing argument to %<target(\"branch-protection=\")%> pragma or"
-	      " attribute");
-       break;
-     case AARCH_PARSE_INVALID_ARG:
-       error ("invalid protection type %qs in %<target(\"branch-protection"
-	      "=\")%> pragma or attribute", err_str);
-       break;
-     case AARCH_PARSE_OK:
-       success = true;
-      /* Fall through.  */
-     case AARCH_PARSE_INVALID_FEATURE:
-       break;
-     default:
-       gcc_unreachable ();
-    }
-  free (err_str);
-  return success;
- }
+static bool
+aarch64_handle_attr_branch_protection (const char* str)
+{
+  return aarch_validate_mbranch_protection (str,
+					    "target(\"branch-protection=\")");
+}
 
 /* Handle the argument STR to the tune= target attribute.  */
 
@@ -22111,6 +22109,19 @@ aarch64_stepped_int_parallel_p (rtx op, int step)
   return true;
 }
 
+/* Return true if OPERANDS[0] to OPERANDS[NUM_OPERANDS - 1] form a
+   sequence of strided registers, with the stride being equal STRIDE.
+   The operands are already known to be FPRs.  */
+bool
+aarch64_strided_registers_p (rtx *operands, unsigned int num_operands,
+			     unsigned int stride)
+{
+  for (unsigned int i = 1; i < num_operands; ++i)
+    if (REGNO (operands[i]) != REGNO (operands[0]) + i * stride)
+      return false;
+  return true;
+}
+
 /* Bounds-check lanes.  Ensure OPERAND lies between LOW (inclusive) and
    HIGH (exclusive).  */
 void
@@ -23896,6 +23907,10 @@ aarch64_float_const_representable_p (rtx x)
       || REAL_VALUE_MINUS_ZERO (r))
     return false;
 
+  /* For BFmode, only handle 0.0. */
+  if (GET_MODE (x) == BFmode)
+    return real_iszero (&r, false);
+
   /* Extract exponent.  */
   r = real_value_abs (&r);
   exponent = REAL_EXP (&r);
@@ -27512,33 +27527,61 @@ supported_simd_type (tree t)
   return false;
 }
 
-/* Return true for types that currently are supported as SIMD return
-   or argument types.  */
+/* Determine the lane size for the clone argument/return type.  This follows
+   the LS(P) rule in the VFABIA64.  */
 
-static bool
-currently_supported_simd_type (tree t, tree b)
+static unsigned
+lane_size (cgraph_simd_clone_arg_type clone_arg_type, tree type)
 {
-  if (COMPLEX_FLOAT_TYPE_P (t))
-    return false;
+  gcc_assert (clone_arg_type != SIMD_CLONE_ARG_TYPE_MASK);
 
-  if (TYPE_SIZE (t) != TYPE_SIZE (b))
-    return false;
+  /* For non map-to-vector types that are pointers we use the element type it
+     points to.  */
+  if (POINTER_TYPE_P (type))
+    switch (clone_arg_type)
+      {
+      default:
+	break;
+      case SIMD_CLONE_ARG_TYPE_UNIFORM:
+      case SIMD_CLONE_ARG_TYPE_LINEAR_CONSTANT_STEP:
+      case SIMD_CLONE_ARG_TYPE_LINEAR_VARIABLE_STEP:
+	type = TREE_TYPE (type);
+	break;
+      }
 
-  return supported_simd_type (t);
+  /* For types (or pointers of non map-to-vector types point to) that are
+     integers or floating point, we use their size if they are 1, 2, 4 or 8.
+   */
+  if (INTEGRAL_TYPE_P (type)
+      || SCALAR_FLOAT_TYPE_P (type))
+    switch (TYPE_PRECISION (type) / BITS_PER_UNIT)
+      {
+      default:
+	break;
+      case 1:
+      case 2:
+      case 4:
+      case 8:
+	return TYPE_PRECISION (type);
+      }
+  /* For any other we use the size of uintptr_t.  For map-to-vector types that
+     are pointers, using the size of uintptr_t is the same as using the size of
+     their type, seeing all pointers are the same size as uintptr_t.  */
+  return POINTER_SIZE;
 }
 
+
 /* Implement TARGET_SIMD_CLONE_COMPUTE_VECSIZE_AND_SIMDLEN.  */
 
 static int
 aarch64_simd_clone_compute_vecsize_and_simdlen (struct cgraph_node *node,
 					struct cgraph_simd_clone *clonei,
-					tree base_type, int num,
-					bool explicit_p)
+					tree base_type ATTRIBUTE_UNUSED,
+					int num, bool explicit_p)
 {
   tree t, ret_type;
-  unsigned int elt_bits, count;
+  unsigned int nds_elt_bits;
   unsigned HOST_WIDE_INT const_simdlen;
-  poly_uint64 vec_bits;
 
   if (!TARGET_SIMD)
     return 0;
@@ -27558,80 +27601,132 @@ aarch64_simd_clone_compute_vecsize_and_simdlen (struct cgraph_node *node,
     }
 
   ret_type = TREE_TYPE (TREE_TYPE (node->decl));
+  /* According to AArch64's Vector ABI the type that determines the simdlen is
+     the narrowest of types, so we ignore base_type for AArch64.  */
   if (TREE_CODE (ret_type) != VOID_TYPE
-      && !currently_supported_simd_type (ret_type, base_type))
+      && !supported_simd_type (ret_type))
     {
       if (!explicit_p)
 	;
-      else if (TYPE_SIZE (ret_type) != TYPE_SIZE (base_type))
-	warning_at (DECL_SOURCE_LOCATION (node->decl), 0,
-		    "GCC does not currently support mixed size types "
-		    "for %<simd%> functions");
-      else if (supported_simd_type (ret_type))
+      else if (COMPLEX_FLOAT_TYPE_P (ret_type))
 	warning_at (DECL_SOURCE_LOCATION (node->decl), 0,
 		    "GCC does not currently support return type %qT "
-		    "for %<simd%> functions", ret_type);
+		    "for simd", ret_type);
       else
 	warning_at (DECL_SOURCE_LOCATION (node->decl), 0,
-		    "unsupported return type %qT for %<simd%> functions",
+		    "unsupported return type %qT for simd",
 		    ret_type);
       return 0;
     }
 
+  auto_vec<std::pair <tree, unsigned int>> vec_elts (clonei->nargs + 1);
+
+  /* We are looking for the NDS type here according to the VFABIA64.  */
+  if (TREE_CODE (ret_type) != VOID_TYPE)
+    {
+      nds_elt_bits = lane_size (SIMD_CLONE_ARG_TYPE_VECTOR, ret_type);
+      vec_elts.safe_push (std::make_pair (ret_type, nds_elt_bits));
+    }
+  else
+    nds_elt_bits = POINTER_SIZE;
+
   int i;
   tree type_arg_types = TYPE_ARG_TYPES (TREE_TYPE (node->decl));
   bool decl_arg_p = (node->definition || type_arg_types == NULL_TREE);
-
   for (t = (decl_arg_p ? DECL_ARGUMENTS (node->decl) : type_arg_types), i = 0;
        t && t != void_list_node; t = TREE_CHAIN (t), i++)
     {
       tree arg_type = decl_arg_p ? TREE_TYPE (t) : TREE_VALUE (t);
-
       if (clonei->args[i].arg_type != SIMD_CLONE_ARG_TYPE_UNIFORM
-	  && !currently_supported_simd_type (arg_type, base_type))
+	  && !supported_simd_type (arg_type))
 	{
 	  if (!explicit_p)
 	    ;
-	  else if (TYPE_SIZE (arg_type) != TYPE_SIZE (base_type))
-	    warning_at (DECL_SOURCE_LOCATION (node->decl), 0,
-			"GCC does not currently support mixed size types "
-			"for %<simd%> functions");
-	  else
+	  else if (COMPLEX_FLOAT_TYPE_P (ret_type))
 	    warning_at (DECL_SOURCE_LOCATION (node->decl), 0,
 			"GCC does not currently support argument type %qT "
-			"for %<simd%> functions", arg_type);
+			"for simd", arg_type);
+	  else
+	    warning_at (DECL_SOURCE_LOCATION (node->decl), 0,
+			"unsupported argument type %qT for simd",
+			arg_type);
 	  return 0;
 	}
+      unsigned lane_bits = lane_size (clonei->args[i].arg_type, arg_type);
+      if (clonei->args[i].arg_type == SIMD_CLONE_ARG_TYPE_VECTOR)
+	vec_elts.safe_push (std::make_pair (arg_type, lane_bits));
+      if (nds_elt_bits > lane_bits)
+	nds_elt_bits = lane_bits;
     }
 
   clonei->vecsize_mangle = 'n';
   clonei->mask_mode = VOIDmode;
-  elt_bits = GET_MODE_BITSIZE (SCALAR_TYPE_MODE (base_type));
+  poly_uint64 simdlen;
+  auto_vec<poly_uint64> simdlens (2);
+  /* Keep track of the possible simdlens the clones of this function can have,
+     and check them later to see if we support them.  */
   if (known_eq (clonei->simdlen, 0U))
     {
-      count = 2;
-      vec_bits = (num == 0 ? 64 : 128);
-      clonei->simdlen = exact_div (vec_bits, elt_bits);
+      simdlen = exact_div (poly_uint64 (64), nds_elt_bits);
+      simdlens.safe_push (simdlen);
+      simdlens.safe_push (simdlen * 2);
     }
   else
+    simdlens.safe_push (clonei->simdlen);
+
+  clonei->vecsize_int = 0;
+  clonei->vecsize_float = 0;
+
+  /* We currently do not support generating simdclones where vector arguments
+     do not fit into a single vector register, i.e. vector types that are more
+     than 128-bits large.  This is because of how we currently represent such
+     types in ACLE, where we use a struct to allow us to pass them as arguments
+     and return.
+     Hence why we have to check whether the simdlens available for this
+     simdclone would cause a vector type to be larger than 128-bits, and reject
+     such a clone.  */
+  unsigned j = 0;
+  while (j < simdlens.length ())
     {
-      count = 1;
-      vec_bits = clonei->simdlen * elt_bits;
-      /* For now, SVE simdclones won't produce illegal simdlen, So only check
-	 const simdlens here.  */
-      if (clonei->simdlen.is_constant (&const_simdlen)
-	  && maybe_ne (vec_bits, 64U) && maybe_ne (vec_bits, 128U))
-	{
-	  if (explicit_p)
-	    warning_at (DECL_SOURCE_LOCATION (node->decl), 0,
-			"GCC does not currently support simdlen %wd for "
-			"type %qT",
-			const_simdlen, base_type);
-	  return 0;
-	}
+      bool remove_simdlen = false;
+      for (auto elt : vec_elts)
+	if (known_gt (simdlens[j] * elt.second, 128U))
+	  {
+	    /* Don't issue a warning for every simdclone when there is no
+	       specific simdlen clause.  */
+	    if (explicit_p && maybe_ne (clonei->simdlen, 0U))
+	      warning_at (DECL_SOURCE_LOCATION (node->decl), 0,
+			  "GCC does not currently support simdlen %wd for "
+			  "type %qT",
+			  constant_lower_bound (simdlens[j]), elt.first);
+	    remove_simdlen = true;
+	    break;
+	  }
+      if (remove_simdlen)
+	simdlens.ordered_remove (j);
+      else
+	j++;
     }
-  clonei->vecsize_int = vec_bits;
-  clonei->vecsize_float = vec_bits;
+
+
+  int count = simdlens.length ();
+  if (count == 0)
+    {
+      if (explicit_p && known_eq (clonei->simdlen, 0U))
+	{
+	  /* Warn the user if we can't generate any simdclone.  */
+	  simdlen = exact_div (poly_uint64 (64), nds_elt_bits);
+	  warning_at (DECL_SOURCE_LOCATION (node->decl), 0,
+		      "GCC does not currently support a simdclone with simdlens"
+		      " %wd and %wd for these types.",
+		      constant_lower_bound (simdlen),
+		      constant_lower_bound (simdlen*2));
+	}
+      return 0;
+    }
+
+  gcc_assert (num < count);
+  clonei->simdlen = simdlens[num];
   return count;
 }
 
@@ -29083,9 +29178,10 @@ aarch64_valid_sysreg_name_p (const char *regname)
 
 /* Return the generic sysreg specification for a valid system register
    name, otherwise NULL.  WRITE_P is true iff the register is being
-   written to.  */
+   written to.  IS128OP indicates the requested system register should
+   be checked for a 128-bit implementation.  */
 const char *
-aarch64_retrieve_sysreg (const char *regname, bool write_p)
+aarch64_retrieve_sysreg (const char *regname, bool write_p, bool is128op)
 {
   const sysreg_t *sysreg = aarch64_lookup_sysreg_map (regname);
   if (sysreg == NULL)
@@ -29095,6 +29191,8 @@ aarch64_retrieve_sysreg (const char *regname, bool write_p)
       else
 	return NULL;
     }
+  if (is128op && !(sysreg->properties & F_REG_128))
+    return NULL;
   if ((write_p && (sysreg->properties & F_REG_READ))
       || (!write_p && (sysreg->properties & F_REG_WRITE)))
     return NULL;

gcc/config/aarch64/aarch64.h

@@ -250,17 +250,23 @@ constexpr auto AARCH64_FL_DEFAULT_ISA_MODE = AARCH64_FL_SM_OFF;
 #define AARCH64_ISA_F64MM	   (aarch64_isa_flags & AARCH64_FL_F64MM)
 #define AARCH64_ISA_BF16	   (aarch64_isa_flags & AARCH64_FL_BF16)
 #define AARCH64_ISA_SB		   (aarch64_isa_flags & AARCH64_FL_SB)
+#define AARCH64_ISA_RCPC3	   (aarch64_isa_flags & AARCH64_FL_RCPC3)
 #define AARCH64_ISA_V8R		   (aarch64_isa_flags & AARCH64_FL_V8R)
 #define AARCH64_ISA_PAUTH	   (aarch64_isa_flags & AARCH64_FL_PAUTH)
 #define AARCH64_ISA_V8_7A	   (aarch64_isa_flags & AARCH64_FL_V8_7A)
 #define AARCH64_ISA_V8_8A	   (aarch64_isa_flags & AARCH64_FL_V8_8A)
+#define AARCH64_ISA_V8_9A	   (aarch64_isa_flags & AARCH64_FL_V8_9A)
 #define AARCH64_ISA_V9A		   (aarch64_isa_flags & AARCH64_FL_V9A)
 #define AARCH64_ISA_V9_1A          (aarch64_isa_flags & AARCH64_FL_V9_1A)
 #define AARCH64_ISA_V9_2A          (aarch64_isa_flags & AARCH64_FL_V9_2A)
 #define AARCH64_ISA_V9_3A          (aarch64_isa_flags & AARCH64_FL_V9_3A)
+#define AARCH64_ISA_V9_4A	   (aarch64_isa_flags & AARCH64_FL_V9_4A)
 #define AARCH64_ISA_MOPS	   (aarch64_isa_flags & AARCH64_FL_MOPS)
 #define AARCH64_ISA_LS64	   (aarch64_isa_flags & AARCH64_FL_LS64)
 #define AARCH64_ISA_CSSC	   (aarch64_isa_flags & AARCH64_FL_CSSC)
+#define AARCH64_ISA_D128	   (aarch64_isa_flags & AARCH64_FL_D128)
+#define AARCH64_ISA_THE		   (aarch64_isa_flags & AARCH64_FL_THE)
+#define AARCH64_ISA_GCS		   (aarch64_isa_flags & AARCH64_FL_GCS)
 
 /* The current function is a normal non-streaming function.  */
 #define TARGET_NON_STREAMING (AARCH64_ISA_SM_OFF)
@@ -428,6 +434,9 @@ constexpr auto AARCH64_FL_DEFAULT_ISA_MODE = AARCH64_FL_SM_OFF;
    and sign-extending versions.*/
 #define TARGET_RCPC2 (AARCH64_ISA_RCPC8_4)
 
+/* RCPC3 (Release Consistency) extensions, optional from Armv8.2-a.  */
+#define TARGET_RCPC3 (AARCH64_ISA_RCPC3)
+
 /* Apply the workaround for Cortex-A53 erratum 835769.  */
 #define TARGET_FIX_ERR_A53_835769	\
   ((aarch64_fix_a53_err835769 == 2)	\
@@ -450,6 +459,22 @@ constexpr auto AARCH64_FL_DEFAULT_ISA_MODE = AARCH64_FL_SM_OFF;
 /* ARMv8.1-A Adv.SIMD support.  */
 #define TARGET_SIMD_RDMA (TARGET_SIMD && AARCH64_ISA_RDMA)
 
+/* Armv9.4-A features.  */
+#define TARGET_ARMV9_4 (AARCH64_ISA_V9_4A)
+
+/*  128-bit System Registers and Instructions from Armv9.4-a are enabled
+    through +d128.  */
+#define TARGET_D128 (AARCH64_ISA_D128)
+
+/*  Armv8.9-A/9.4-A Translation Hardening Extension system registers are
+    enabled through +the.  */
+#define TARGET_THE (AARCH64_ISA_THE)
+
+/*  Armv9.4-A Guarded Control Stack extension system registers are
+    enabled through +gcs.  */
+#define TARGET_GCS (AARCH64_ISA_GCS)
+
+
 /* Standard register usage.  */
 
 /* 31 64-bit general purpose registers R0-R30:

gcc/config/aarch64/aarch64.md

@@ -290,13 +290,9 @@
     UNSPEC_NZCV
     UNSPEC_XPACLRI
     UNSPEC_LD1_SVE
-    UNSPEC_LD1_SVE_COUNT
     UNSPEC_ST1_SVE
-    UNSPEC_ST1_SVE_COUNT
     UNSPEC_LDNT1_SVE
-    UNSPEC_LDNT1_SVE_COUNT
     UNSPEC_STNT1_SVE
-    UNSPEC_STNT1_SVE_COUNT
     UNSPEC_LD1RQ
     UNSPEC_LD1_GATHER
     UNSPEC_LDFF1_GATHER
@@ -339,7 +335,10 @@
     UNSPEC_RDFFR
     UNSPEC_WRFFR
     UNSPEC_SYSREG_RDI
+    UNSPEC_SYSREG_RTI
     UNSPEC_SYSREG_WDI
+    UNSPEC_SYSREG_WTI
+    UNSPEC_PLDX
     ;; Represents an SVE-style lane index, in which the indexing applies
     ;; within the containing 128-bit block.
     UNSPEC_SVE_LANE_SELECT
@@ -356,6 +355,8 @@
     UNSPEC_SAVE_NZCV
     UNSPEC_RESTORE_NZCV
     UNSPECV_PATCHABLE_AREA
+    UNSPEC_LDAP1_LANE
+    UNSPEC_STL1_LANE
     ;; Wraps a constant integer that should be multiplied by the number
     ;; of quadwords in an SME vector.
     UNSPEC_SME_VQ
@@ -527,6 +528,26 @@
 ;; may chose to hold the tracking state encoded in SP.
 (define_attr "speculation_barrier" "true,false" (const_string "false"))
 
+;; This attribute is attached to multi-register instructions that have
+;; two forms: one in which the registers are consecutive and one in
+;; which they are strided.  The consecutive and strided forms have
+;; different define_insns, with different operands.  The mapping between
+;; the RTL of the consecutive form and the RTL of the strided form varies
+;; from one type of instruction to another.
+;;
+;; The attribute gives two pieces of information:
+;; - does the current instruction have consecutive or strided registers?
+;; - what kind of RTL rewrite is needed to move between forms?
+;;
+;; For example, all consecutive LD*1 instructions have the same basic
+;; RTL structure.  The same applies to all strided LD*1 instructions.
+;; The RTL mapping therefore applies at LD1 granularity, rather than
+;; being broken down into individual types of load.
+(define_attr "stride_type"
+  "none,ld1_consecutive,ld1_strided,st1_consecutive,st1_strided,
+   luti_consecutive,luti_strided"
+  (const_string "none"))
+
 ;; -------------------------------------------------------------------
 ;; Pipeline descriptions and scheduling
 ;; -------------------------------------------------------------------
@@ -558,6 +579,14 @@
   "mrs\t%x0, %1"
 )
 
+(define_insn "aarch64_read_sysregti"
+  [(set (match_operand:TI 0 "register_operand" "=r")
+    (unspec_volatile:TI [(match_operand 1 "aarch64_sysreg_string" "")]
+			UNSPEC_SYSREG_RTI))]
+ "TARGET_D128"
+ "mrrs\t%x0, %H0, %x1"
+)
+
 (define_insn "aarch64_write_sysregdi"
   [(unspec_volatile:DI [(match_operand 0 "aarch64_sysreg_string" "")
 			(match_operand:DI 1 "register_operand" "rZ")]
@@ -566,6 +595,14 @@
   "msr\t%0, %x1"
 )
 
+(define_insn "aarch64_write_sysregti"
+ [(unspec_volatile:TI [(match_operand 0 "aarch64_sysreg_string" "")
+		       (match_operand:TI 1 "register_operand" "r")]
+		      UNSPEC_SYSREG_WTI)]
+ "TARGET_D128"
+ "msrr\t%x0, %x1, %H1"
+)
+
 (define_insn "indirect_jump"
   [(set (pc) (match_operand:DI 0 "register_operand" "r"))]
   ""
@@ -934,6 +971,17 @@
   [(set_attr "type" "load_4")]
 )
 
+(define_insn "aarch64_pldx"
+  [(unspec [(match_operand 0 "" "")
+	    (match_operand:DI 1 "aarch64_prefetch_operand" "Dp")] UNSPEC_PLDX)]
+  ""
+  {
+    operands[1] = gen_rtx_MEM (DImode, operands[1]);
+    return "prfm\\t%0, %1";
+  }
+  [(set_attr "type" "load_4")]
+)
+
 (define_insn "trap"
   [(trap_if (const_int 1) (const_int 8))]
   ""

gcc/config/aarch64/aarch64.opt

@@ -237,6 +237,24 @@ Enable the division approximation.  Enabling this reduces
 precision of division results to about 16 bits for
 single precision and to 32 bits for double precision.
 
+Enum
+Name(early_ra_scope) Type(enum aarch64_early_ra_scope)
+
+EnumValue
+Enum(early_ra_scope) String(all) Value(AARCH64_EARLY_RA_ALL)
+
+EnumValue
+Enum(early_ra_scope) String(strided) Value(AARCH64_EARLY_RA_STRIDED)
+
+EnumValue
+Enum(early_ra_scope) String(none) Value(AARCH64_EARLY_RA_NONE)
+
+mearly-ra=
+Target RejectNegative Joined Enum(early_ra_scope) Var(aarch64_early_ra) Init(AARCH64_EARLY_RA_NONE) Save
+Specify when to enable an early register allocation pass.  The possibilities
+are: all functions, functions that have access to strided multi-register
+instructions, and no functions.
+
 Enum
 Name(sve_vector_bits) Type(enum aarch64_sve_vector_bits_enum)
 The possible SVE vector lengths:

gcc/config/aarch64/arm_acle.h

@@ -78,6 +78,36 @@ _GCC_ARM_ACLE_DATA_FN (revll, bswap64, uint64_t, uint64_t)
 
 #undef _GCC_ARM_ACLE_DATA_FN
 
+__extension__ extern __inline void
+__attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
+__pld (void const volatile *__addr)
+{
+  return __builtin_aarch64_pld (__addr);
+}
+
+__extension__ extern __inline void
+__attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
+__pli (void const volatile *__addr)
+{
+  return __builtin_aarch64_pli (__addr);
+}
+
+__extension__ extern __inline void
+__attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
+__plix (unsigned int __cache, unsigned int __rettn,
+	void const volatile *__addr)
+{
+  return __builtin_aarch64_plix (__cache, __rettn, __addr);
+}
+
+__extension__ extern __inline void
+__attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
+__pldx (unsigned int __access, unsigned int __cache, unsigned int __rettn,
+	void const volatile *__addr)
+{
+  return __builtin_aarch64_pldx (__access, __cache, __rettn, __addr);
+}
+
 __extension__ extern __inline unsigned long
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 __revl (unsigned long __value)
@@ -344,6 +374,17 @@ __rndrrs (uint64_t *__res)
 #define __arm_wsrf64(__regname, __value) \
   __builtin_aarch64_wsrf64 (__regname, __value)
 
+#pragma GCC push_options
+#pragma GCC target ("+nothing+d128")
+
+#define __arm_rsr128(__regname) \
+  __builtin_aarch64_rsr128 (__regname)
+
+#define __arm_wsr128(__regname, __value) \
+  __builtin_aarch64_wsr128 (__regname, __value)
+
+#pragma GCC pop_options
+
 #ifdef __cplusplus
 }
 #endif

gcc/config/aarch64/arm_neon.h

@@ -13446,6 +13446,143 @@ vld1q_lane_u64 (const uint64_t *__src, uint64x2_t __vec, const int __lane)
   return __aarch64_vset_lane_any (*__src, __vec, __lane);
 }
 
+#pragma GCC push_options
+#pragma GCC target ("+nothing+rcpc3+simd")
+
+/* vldap1_lane.  */
+
+__extension__ extern __inline uint64x1_t
+__attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
+vldap1_lane_u64 (const uint64_t *__src, uint64x1_t __vec, const int __lane)
+{
+  return __builtin_aarch64_vec_ldap1_lanev1di_usus (
+	  (const __builtin_aarch64_simd_di *) __src, __vec, __lane);
+}
+
+__extension__ extern __inline uint64x2_t
+__attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
+vldap1q_lane_u64 (const uint64_t *__src, uint64x2_t __vec, const int __lane)
+{
+  return __builtin_aarch64_vec_ldap1_lanev2di_usus (
+	  (const __builtin_aarch64_simd_di *) __src, __vec, __lane);
+}
+
+__extension__ extern __inline int64x1_t
+__attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
+vldap1_lane_s64 (const int64_t *__src, int64x1_t __vec, const int __lane)
+{
+  return __builtin_aarch64_vec_ldap1_lanev1di (
+	  (const __builtin_aarch64_simd_di *) __src, __vec, __lane);
+}
+
+__extension__ extern __inline int64x2_t
+__attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
+vldap1q_lane_s64 (const int64_t *__src, int64x2_t __vec, const int __lane)
+{
+  return __builtin_aarch64_vec_ldap1_lanev2di (
+	  (const __builtin_aarch64_simd_di *) __src, __vec, __lane);
+}
+
+__extension__ extern __inline float64x1_t
+__attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
+vldap1_lane_f64 (const float64_t *__src, float64x1_t __vec, const int __lane)
+{
+  return __builtin_aarch64_vec_ldap1_lanev1df (
+	  (const __builtin_aarch64_simd_df *) __src, __vec, __lane);
+}
+
+__extension__ extern __inline float64x2_t
+__attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
+vldap1q_lane_f64 (const float64_t *__src, float64x2_t __vec, const int __lane)
+{
+  return __builtin_aarch64_vec_ldap1_lanev2df (
+	  (const __builtin_aarch64_simd_df *) __src, __vec, __lane);
+}
+
+__extension__ extern __inline poly64x1_t
+__attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
+vldap1_lane_p64 (const poly64_t *__src, poly64x1_t __vec, const int __lane)
+{
+  return __builtin_aarch64_vec_ldap1_lanev1di_psps (
+	  (const __builtin_aarch64_simd_di *) __src, __vec, __lane);
+}
+
+__extension__ extern __inline poly64x2_t
+__attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
+vldap1q_lane_p64 (const poly64_t *__src, poly64x2_t __vec, const int __lane)
+{
+  return __builtin_aarch64_vec_ldap1_lanev2di_psps (
+	  (const __builtin_aarch64_simd_di *) __src, __vec, __lane);
+}
+
+/* vstl1_lane.  */
+
+__extension__ extern __inline void
+__attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
+vstl1_lane_u64 (uint64_t *__src, uint64x1_t __vec, const int __lane)
+{
+  __builtin_aarch64_vec_stl1_lanev1di_sus ((__builtin_aarch64_simd_di *) __src,
+					   __vec, __lane);
+}
+
+__extension__ extern __inline void
+__attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
+vstl1q_lane_u64 (uint64_t *__src, uint64x2_t __vec, const int __lane)
+{
+  __builtin_aarch64_vec_stl1_lanev2di_sus ((__builtin_aarch64_simd_di *) __src,
+					   __vec, __lane);
+}
+
+__extension__ extern __inline void
+__attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
+vstl1_lane_s64 (int64_t *__src, int64x1_t __vec, const int __lane)
+{
+  __builtin_aarch64_vec_stl1_lanev1di ((__builtin_aarch64_simd_di *) __src,
+				       __vec, __lane);
+}
+
+__extension__ extern __inline void
+__attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
+vstl1q_lane_s64 (int64_t *__src, int64x2_t __vec, const int __lane)
+{
+  __builtin_aarch64_vec_stl1_lanev2di ((__builtin_aarch64_simd_di *) __src,
+				       __vec, __lane);
+}
+
+__extension__ extern __inline void
+__attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
+vstl1_lane_f64 (float64_t *__src, float64x1_t __vec, const int __lane)
+{
+  __builtin_aarch64_vec_stl1_lanev1df ((__builtin_aarch64_simd_df *) __src,
+				       __vec, __lane);
+}
+
+__extension__ extern __inline void
+__attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
+vstl1q_lane_f64 (float64_t *__src, float64x2_t __vec, const int __lane)
+{
+  __builtin_aarch64_vec_stl1_lanev2df ((__builtin_aarch64_simd_df *) __src,
+				       __vec, __lane);
+}
+
+__extension__ extern __inline void
+__attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
+vstl1_lane_p64 (poly64_t *__src, poly64x1_t __vec, const int __lane)
+{
+  __builtin_aarch64_vec_stl1_lanev1di_sps ((__builtin_aarch64_simd_di *) __src,
+					   __vec, __lane);
+}
+
+__extension__ extern __inline void
+__attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
+vstl1q_lane_p64 (poly64_t *__src, poly64x2_t __vec, const int __lane)
+{
+  __builtin_aarch64_vec_stl1_lanev2di_sps ((__builtin_aarch64_simd_di *) __src,
+					   __vec, __lane);
+}
+
+#pragma GCC pop_options
+
 /* vldn */
 
 __extension__ extern __inline int64x1x2_t

gcc/config/aarch64/arm_neon_sve_bridge.h

@@ -0,0 +1,38 @@
+/* AArch64 NEON-SVE Bridge intrinsics include file.
+   Copyright (C) 2023 Free Software Foundation, Inc.
+
+   This file is part of GCC.
+
+   GCC is free software; you can redistribute it and/or modify it
+   under the terms of the GNU General Public License as published
+   by the Free Software Foundation; either version 3, or (at your
+   option) any later version.
+
+   GCC is distributed in the hope that it will be useful, but WITHOUT
+   ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
+   or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public
+   License for more details.
+
+   Under Section 7 of GPL version 3, you are granted additional
+   permissions described in the GCC Runtime Library Exception, version
+   3.1, as published by the Free Software Foundation.
+
+   You should have received a copy of the GNU General Public License and
+   a copy of the GCC Runtime Library Exception along with this program;
+   see the files COPYING3 and COPYING.RUNTIME respectively.  If not, see
+   <http://www.gnu.org/licenses/>.  */
+
+#ifndef _ARM_NEON_SVE_BRIDGE_H_
+#define _ARM_NEON_SVE_BRIDGE_H_
+
+#include <arm_neon.h>
+#include <arm_sve.h>
+
+/* NOTE: This implementation of arm_neon_sve_bridge.h is intentionally short.  It does
+   not define the types and intrinsic functions directly in C and C++
+   code, but instead uses the following pragma to tell GCC to insert the
+   necessary type and function definitions itself.  The net effect is the
+   same, and the file is a complete implementation of arm_neon_sve_bridge.h.  */
+#pragma GCC aarch64 "arm_neon_sve_bridge.h"
+
+#endif

gcc/config/aarch64/constraints.md

@@ -56,6 +56,14 @@
   "4-tuple-aligned floating point and SIMD vector registers."
   "regno % 4 == 0")
 
+(define_register_constraint "Uwd" "FP_REGS"
+  "@internal The first register in a tuple of 2 strided FPRs."
+  "(regno & 0x8) == 0")
+
+(define_register_constraint "Uwt" "FP_REGS"
+  "@internal The first register in a tuple of 4 strided FPRs."
+  "(regno & 0xc) == 0")
+
 (define_register_constraint "Upa" "PR_REGS"
   "SVE predicate registers p0 - p15.")
 

gcc/config/aarch64/iterators.md

@@ -314,6 +314,12 @@
 ;; All byte modes.
 (define_mode_iterator VB [V8QI V16QI])
 
+;; 1 and 2 lane DI and DF modes.
+(define_mode_iterator V12DIF [V1DI V1DF V2DI V2DF])
+
+;; 1 and 2 lane DI mode.
+(define_mode_iterator V12DI [V1DI V2DI])
+
 ;; 2 and 4 lane SI modes.
 (define_mode_iterator VS [V2SI V4SI])
 
@@ -800,6 +806,7 @@
     UNSPEC_FTSMUL	; Used in aarch64-sve.md.
     UNSPEC_FTSSEL	; Used in aarch64-sve.md.
     UNSPEC_SMATMUL	; Used in aarch64-sve.md.
+    UNSPEC_SET_NEONQ	; Used in aarch64-sve.md.
     UNSPEC_UMATMUL	; Used in aarch64-sve.md.
     UNSPEC_USMATMUL	; Used in aarch64-sve.md.
     UNSPEC_TRN1Q	; Used in aarch64-sve.md.
@@ -920,6 +927,8 @@
     UNSPEC_FMLSLT	; Used in aarch64-sve2.md.
     UNSPEC_HISTCNT	; Used in aarch64-sve2.md.
     UNSPEC_HISTSEG	; Used in aarch64-sve2.md.
+    UNSPEC_LD1_COUNT	; Used in aarch64-sve2.md.
+    UNSPEC_LDNT1_COUNT	; Used in aarch64-sve2.md.
     UNSPEC_MATCH	; Used in aarch64-sve2.md.
     UNSPEC_NMATCH	; Used in aarch64-sve2.md.
     UNSPEC_PEXT		; Used in aarch64-sve2.md.
@@ -988,6 +997,8 @@
     UNSPEC_SSUBLTB	; Used in aarch64-sve2.md.
     UNSPEC_SSUBWB	; Used in aarch64-sve2.md.
     UNSPEC_SSUBWT	; Used in aarch64-sve2.md.
+    UNSPEC_ST1_COUNT	; Used in aarch64-sve2.md.
+    UNSPEC_STNT1_COUNT	; Used in aarch64-sve2.md.
     UNSPEC_SUBHNB	; Used in aarch64-sve2.md.
     UNSPEC_SUBHNT	; Used in aarch64-sve2.md.
     UNSPEC_TBL2		; Used in aarch64-sve2.md.
@@ -1324,10 +1335,10 @@
 (define_mode_attr Vetype [(V8QI "b") (V16QI "b")
 			  (V4HI "h") (V8HI  "h")
 			  (V2SI "s") (V4SI  "s")
-			  (V2DI "d")
+			  (V2DI "d") (V1DI  "d")
 			  (V4HF "h") (V8HF  "h")
 			  (V2SF "s") (V4SF  "s")
-			  (V2DF "d")
+			  (V2DF "d") (V1DF  "d")
 			  (V2x8QI "b") (V2x4HI "h")
 			  (V2x2SI "s") (V2x1DI "d")
 			  (V2x4HF "h") (V2x2SF "s")
@@ -1498,10 +1509,12 @@
 (define_mode_attr VEL [(V8QI  "QI") (V16QI "QI")
 		       (V4HI "HI") (V8HI  "HI")
 		       (V2SI "SI") (V4SI  "SI")
-		       (DI   "DI") (V2DI  "DI")
+		       (DI   "DI") (V1DI  "DI")
+		       (V2DI "DI")
 		       (V4HF "HF") (V8HF  "HF")
 		       (V2SF "SF") (V4SF  "SF")
-		       (DF   "DF") (V2DF  "DF")
+		       (DF   "DF") (V1DF  "DF")
+		       (V2DF "DF")
 		       (SI   "SI") (HI    "HI")
 		       (QI   "QI")
 		       (V4BF "BF") (V8BF "BF")
@@ -1518,12 +1531,13 @@
 (define_mode_attr Vel [(V8QI "qi") (V16QI "qi")
 		       (V4HI "hi") (V8HI "hi")
 		       (V2SI "si") (V4SI "si")
-		       (DI   "di") (V2DI "di")
+		       (DI   "di") (V1DI "si")
+		       (V2DI "di")
 		       (V4HF "hf") (V8HF "hf")
 		       (V2SF "sf") (V4SF "sf")
-		       (V2DF "df") (DF   "df")
-		       (SI   "si") (HI   "hi")
-		       (QI   "qi")
+		       (V1DF "df") (V2DF "df")
+		       (DF   "df") (SI   "si")
+		       (HI   "hi") (QI   "qi")
 		       (V4BF "bf") (V8BF "bf")
 		       (VNx16QI "qi") (VNx8QI "qi") (VNx4QI "qi") (VNx2QI "qi")
 		       (VNx8HI "hi") (VNx4HI "hi") (VNx2HI "hi")
@@ -3154,6 +3168,10 @@
 
 (define_int_attr pred_load [(UNSPEC_PRED_X "_x") (UNSPEC_LD1_SVE "")])
 
+(define_int_iterator LD1_COUNT [UNSPEC_LD1_COUNT UNSPEC_LDNT1_COUNT])
+
+(define_int_iterator ST1_COUNT [UNSPEC_ST1_COUNT UNSPEC_STNT1_COUNT])
+
 (define_int_iterator SVE2_U32_UNARY [UNSPEC_URECPE UNSPEC_RSQRTE])
 
 (define_int_iterator SVE2_INT_UNARY_NARROWB [UNSPEC_SQXTNB
@@ -3569,6 +3587,8 @@
 			(UNSPEC_FEXPA "fexpa")
 			(UNSPEC_FTSMUL "ftsmul")
 			(UNSPEC_FTSSEL "ftssel")
+			(UNSPEC_LD1_COUNT "ld1")
+			(UNSPEC_LDNT1_COUNT "ldnt1")
 			(UNSPEC_PMULLB "pmullb")
 			(UNSPEC_PMULLB_PAIR "pmullb_pair")
 			(UNSPEC_PMULLT "pmullt")
@@ -3632,6 +3652,8 @@
 			(UNSPEC_SQRDCMLAH90 "sqrdcmlah90")
 			(UNSPEC_SQRDCMLAH180 "sqrdcmlah180")
 			(UNSPEC_SQRDCMLAH270 "sqrdcmlah270")
+			(UNSPEC_ST1_COUNT "st1")
+			(UNSPEC_STNT1_COUNT "stnt1")
 			(UNSPEC_TRN1Q "trn1q")
 			(UNSPEC_TRN2Q "trn2q")
 			(UNSPEC_UMATMUL "umatmul")

gcc/config/aarch64/t-aarch64

@@ -194,6 +194,12 @@ aarch64-cc-fusion.o: $(srcdir)/config/aarch64/aarch64-cc-fusion.cc \
 	$(COMPILER) -c $(ALL_COMPILERFLAGS) $(ALL_CPPFLAGS) $(INCLUDES) \
 		$(srcdir)/config/aarch64/aarch64-cc-fusion.cc
 
+aarch64-early-ra.o: $(srcdir)/config/aarch64/aarch64-early-ra.cc \
+    $(CONFIG_H) $(SYSTEM_H) $(CORETYPES_H) $(BACKEND_H) $(RTL_H) $(DF_H) \
+    $(RTL_SSA_H) tree-pass.h
+	$(COMPILER) -c $(ALL_COMPILERFLAGS) $(ALL_CPPFLAGS) $(INCLUDES) \
+		$(srcdir)/config/aarch64/aarch64-early-ra.cc
+
 comma=,
 MULTILIB_OPTIONS    = $(subst $(comma),/, $(patsubst %, mabi=%, $(subst $(comma),$(comma)mabi=,$(TM_MULTILIB_CONFIG))))
 MULTILIB_DIRNAMES   = $(subst $(comma), ,$(TM_MULTILIB_CONFIG))

gcc/config/aarch64/x-aarch64

@@ -1,3 +1,5 @@
 driver-aarch64.o: $(srcdir)/config/aarch64/driver-aarch64.cc \
-  $(CONFIG_H) $(SYSTEM_H)
+  $(CONFIG_H) $(SYSTEM_H) $(TM_H) $(CORETYPES_H) \
+  $(srcdir)/config/aarch64/aarch64-protos.h \
+  $(srcdir)/config/aarch64/aarch64-feature-deps.h
 	$(COMPILER) -c $(ALL_COMPILERFLAGS) $(ALL_CPPFLAGS) $(INCLUDES) $<

gcc/config/arc/arc.md

@@ -6127,6 +6127,26 @@ archs4x, archs4xd"
   ""
   [(set_attr "length" "8")])
 
+(define_insn_and_split "*extvsi_n_0"
+  [(set (match_operand:SI 0 "register_operand" "=r")
+	(sign_extract:SI (match_operand:SI 1 "register_operand" "0")
+			 (match_operand:QI 2 "const_int_operand")
+			 (const_int 0)))]
+  "!TARGET_BARREL_SHIFTER
+   && IN_RANGE (INTVAL (operands[2]), 2,
+		(optimize_insn_for_size_p () ? 28 : 30))"
+  "#"
+  "&& 1"
+[(set (match_dup 0) (and:SI (match_dup 0) (match_dup 3)))
+ (set (match_dup 0) (xor:SI (match_dup 0) (match_dup 4)))
+ (set (match_dup 0) (minus:SI (match_dup 0) (match_dup 4)))]
+{
+  int tmp = INTVAL (operands[2]);
+  operands[3] = GEN_INT (~(HOST_WIDE_INT_M1U << tmp));
+  operands[4] = GEN_INT (HOST_WIDE_INT_1U << (tmp - 1));
+}
+  [(set_attr "length" "14")])
+
 (define_insn_and_split "rotlsi3_cnt1"
   [(set (match_operand:SI 0 "dest_reg_operand"            "=r")
 	(rotate:SI (match_operand:SI 1 "register_operand" "r")

gcc/config/arm/aarch-common-protos.h

@@ -159,10 +159,7 @@ rtx_insn *arm_md_asm_adjust (vec<rtx> &outputs, vec<rtx> & /*inputs*/,
 			     vec<rtx> &clobbers, HARD_REG_SET &clobbered_regs,
 			     location_t loc);
 
-/* Parsing routine for branch-protection common to AArch64 and Arm.  */
-enum aarch_parse_opt_result aarch_parse_branch_protection (const char*, char**);
-
 /* Validation routine for branch-protection common to AArch64 and Arm.  */
-bool aarch_validate_mbranch_protection (const char *);
+bool aarch_validate_mbranch_protection (const char *, const char *);
 
 #endif /* GCC_AARCH_COMMON_PROTOS_H */

gcc/config/arm/aarch-common.cc

@@ -660,185 +660,146 @@ arm_md_asm_adjust (vec<rtx> &outputs, vec<rtx> & /*inputs*/,
   return saw_asm_flag ? seq : NULL;
 }
 
-#define BRANCH_PROTECT_STR_MAX 255
-extern char *accepted_branch_protection_string;
-
-static enum aarch_parse_opt_result
-aarch_handle_no_branch_protection (char* str, char* rest)
+static void
+aarch_handle_no_branch_protection (void)
 {
   aarch_ra_sign_scope = AARCH_FUNCTION_NONE;
   aarch_enable_bti = 0;
-  if (rest)
-    {
-      error ("unexpected %<%s%> after %<%s%>", rest, str);
-      return AARCH_PARSE_INVALID_FEATURE;
-    }
-  return AARCH_PARSE_OK;
 }
 
-static enum aarch_parse_opt_result
-aarch_handle_standard_branch_protection (char* str, char* rest)
+static void
+aarch_handle_standard_branch_protection (void)
 {
   aarch_ra_sign_scope = AARCH_FUNCTION_NON_LEAF;
   aarch_ra_sign_key = AARCH_KEY_A;
   aarch_enable_bti = 1;
-  if (rest)
-    {
-      error ("unexpected %<%s%> after %<%s%>", rest, str);
-      return AARCH_PARSE_INVALID_FEATURE;
-    }
-  return AARCH_PARSE_OK;
 }
 
-static enum aarch_parse_opt_result
-aarch_handle_pac_ret_protection (char* str ATTRIBUTE_UNUSED,
-				 char* rest ATTRIBUTE_UNUSED)
+static void
+aarch_handle_pac_ret_protection (void)
 {
   aarch_ra_sign_scope = AARCH_FUNCTION_NON_LEAF;
   aarch_ra_sign_key = AARCH_KEY_A;
-  return AARCH_PARSE_OK;
 }
 
-static enum aarch_parse_opt_result
-aarch_handle_pac_ret_leaf (char* str ATTRIBUTE_UNUSED,
-			   char* rest ATTRIBUTE_UNUSED)
+static void
+aarch_handle_pac_ret_leaf (void)
 {
   aarch_ra_sign_scope = AARCH_FUNCTION_ALL;
-  return AARCH_PARSE_OK;
 }
 
-static enum aarch_parse_opt_result
-aarch_handle_pac_ret_b_key (char* str ATTRIBUTE_UNUSED,
-			    char* rest ATTRIBUTE_UNUSED)
+static void
+aarch_handle_pac_ret_b_key (void)
 {
   aarch_ra_sign_key = AARCH_KEY_B;
-  return AARCH_PARSE_OK;
 }
 
-static enum aarch_parse_opt_result
-aarch_handle_bti_protection (char* str ATTRIBUTE_UNUSED,
-			     char* rest ATTRIBUTE_UNUSED)
+static void
+aarch_handle_bti_protection (void)
 {
   aarch_enable_bti = 1;
-  return AARCH_PARSE_OK;
 }
 
 static const struct aarch_branch_protect_type aarch_pac_ret_subtypes[] = {
-  { "leaf", aarch_handle_pac_ret_leaf, NULL, 0 },
-  { "b-key", aarch_handle_pac_ret_b_key, NULL, 0 },
-  { NULL, NULL, NULL, 0 }
+  { "leaf", false, aarch_handle_pac_ret_leaf, NULL, 0 },
+  { "b-key", false, aarch_handle_pac_ret_b_key, NULL, 0 },
+  { NULL, false, NULL, NULL, 0 }
 };
 
 static const struct aarch_branch_protect_type aarch_branch_protect_types[] = {
-  { "none", aarch_handle_no_branch_protection, NULL, 0 },
-  { "standard", aarch_handle_standard_branch_protection, NULL, 0 },
-  { "pac-ret", aarch_handle_pac_ret_protection, aarch_pac_ret_subtypes,
+  { "none", true, aarch_handle_no_branch_protection, NULL, 0 },
+  { "standard", true, aarch_handle_standard_branch_protection, NULL, 0 },
+  { "pac-ret", false, aarch_handle_pac_ret_protection, aarch_pac_ret_subtypes,
     ARRAY_SIZE (aarch_pac_ret_subtypes) },
-  { "bti", aarch_handle_bti_protection, NULL, 0 },
-  { NULL, NULL, NULL, 0 }
+  { "bti", false, aarch_handle_bti_protection, NULL, 0 },
+  { NULL, false, NULL, NULL, 0 }
 };
 
-/* Parses CONST_STR for branch protection features specified in
-   aarch64_branch_protect_types, and set any global variables required.  Returns
-   the parsing result and assigns LAST_STR to the last processed token from
-   CONST_STR so that it can be used for error reporting.  */
+/* In-place split *str at delim, return *str and set *str to the tail
+   of the string or NULL if the end is reached.  */
 
-enum aarch_parse_opt_result
-aarch_parse_branch_protection (const char *const_str, char** last_str)
+static char *
+next_tok (char **str, int delim)
 {
-  char *str_root = xstrdup (const_str);
-  char* token_save = NULL;
-  char *str = strtok_r (str_root, "+", &token_save);
-  enum aarch_parse_opt_result res = AARCH_PARSE_OK;
-  if (!str)
-    res = AARCH_PARSE_MISSING_ARG;
-  else
+  char *tok = *str;
+  for (char *p = tok; p && *p != '\0'; p++)
     {
-      char *next_str = strtok_r (NULL, "+", &token_save);
-      /* Reset the branch protection features to their defaults.  */
-      aarch_handle_no_branch_protection (NULL, NULL);
-
-      while (str && res == AARCH_PARSE_OK)
+      if (*p == delim)
 	{
-	  const aarch_branch_protect_type* type = aarch_branch_protect_types;
-	  bool found = false;
-	  /* Search for this type.  */
-	  while (type && type->name && !found && res == AARCH_PARSE_OK)
-	    {
-	      if (strcmp (str, type->name) == 0)
-		{
-		  found = true;
-		  res = type->handler (str, next_str);
-		  str = next_str;
-		  next_str = strtok_r (NULL, "+", &token_save);
-		}
-	      else
-		type++;
-	    }
-	  if (found && res == AARCH_PARSE_OK)
-	    {
-	      bool found_subtype = true;
-	      /* Loop through each token until we find one that isn't a
-		 subtype.  */
-	      while (found_subtype)
-		{
-		  found_subtype = false;
-		  const aarch_branch_protect_type *subtype = type->subtypes;
-		  /* Search for the subtype.  */
-		  while (str && subtype && subtype->name && !found_subtype
-			  && res == AARCH_PARSE_OK)
-		    {
-		      if (strcmp (str, subtype->name) == 0)
-			{
-			  found_subtype = true;
-			  res = subtype->handler (str, next_str);
-			  str = next_str;
-			  next_str = strtok_r (NULL, "+", &token_save);
-			}
-		      else
-			subtype++;
-		    }
-		}
-	    }
-	  else if (!found)
-	    res = AARCH_PARSE_INVALID_ARG;
+	  *p = '\0';
+	  *str = p + 1;
+	  return tok;
 	}
     }
-  /* Copy the last processed token into the argument to pass it back.
-    Used by option and attribute validation to print the offending token.  */
-  if (last_str)
-    {
-      if (str)
-	strcpy (*last_str, str);
-      else
-	*last_str = NULL;
-    }
-
-  if (res == AARCH_PARSE_OK)
-    {
-      /* If needed, alloc the accepted string then copy in const_str.
-	Used by override_option_after_change_1.  */
-      if (!accepted_branch_protection_string)
-	accepted_branch_protection_string
-	  = (char *) xmalloc (BRANCH_PROTECT_STR_MAX + 1);
-      strncpy (accepted_branch_protection_string, const_str,
-	       BRANCH_PROTECT_STR_MAX + 1);
-      /* Forcibly null-terminate.  */
-      accepted_branch_protection_string[BRANCH_PROTECT_STR_MAX] = '\0';
-    }
-  return res;
+  *str = NULL;
+  return tok;
 }
 
+/* Parses CONST_STR for branch protection features specified in
+   aarch64_branch_protect_types, and set any global variables required.
+   Returns true on success.  */
+
 bool
-aarch_validate_mbranch_protection (const char *const_str)
+aarch_validate_mbranch_protection (const char *const_str, const char *opt)
 {
-  char *str = (char *) xmalloc (strlen (const_str));
-  enum aarch_parse_opt_result res =
-    aarch_parse_branch_protection (const_str, &str);
-  if (res == AARCH_PARSE_INVALID_ARG)
-    error ("invalid argument %<%s%> for %<-mbranch-protection=%>", str);
-  else if (res == AARCH_PARSE_MISSING_ARG)
-    error ("missing argument for %<-mbranch-protection=%>");
-  free (str);
-  return res == AARCH_PARSE_OK;
+  char *str_root = xstrdup (const_str);
+  char *next_str = str_root;
+  char *str = next_tok (&next_str, '+');
+  char *alone_str = NULL;
+  bool reject_alone = false;
+  bool res = true;
+
+  /* First entry is "none" and it is used to reset the state.  */
+  aarch_branch_protect_types[0].handler ();
+
+  while (str)
+    {
+      const aarch_branch_protect_type *type = aarch_branch_protect_types;
+      for (; type->name; type++)
+	if (strcmp (str, type->name) == 0)
+	  break;
+      if (type->name == NULL)
+	{
+	  res = false;
+	  if (strcmp (str, "") == 0)
+	    error ("missing feature or flag for %<%s%>", opt);
+	  else
+	    error ("invalid argument %<%s%> for %<%s%>", str, opt);
+	  break;
+	}
+
+      if (type->alone && alone_str == NULL)
+	alone_str = str;
+      else
+	reject_alone = true;
+      if (reject_alone && alone_str != NULL)
+	{
+	  res = false;
+	  error ("argument %<%s%> can only appear alone in %<%s%>",
+		 alone_str, opt);
+	  break;
+	}
+
+      type->handler ();
+      str = next_tok (&next_str, '+');
+      if (type->subtypes == NULL)
+	continue;
+
+      /* Loop through tokens until we find one that isn't a subtype.  */
+      while (str)
+	{
+	  const aarch_branch_protect_type *subtype = type->subtypes;
+	  for (; subtype->name; subtype++)
+	    if (strcmp (str, subtype->name) == 0)
+	      break;
+	  if (subtype->name == NULL)
+	    break;
+
+	  subtype->handler ();
+	  str = next_tok (&next_str, '+');
+	}
+    }
+
+  free (str_root);
+  return res;
 }

gcc/config/arm/aarch-common.h

@@ -55,16 +55,10 @@ struct aarch_branch_protect_type
   /* The type's name that the user passes to the branch-protection option
      string.  */
   const char* name;
-  /* Function to handle the protection type and set global variables.
-     First argument is the string token corresponding with this type and the
-     second argument is the next token in the option string.
-     Return values:
-     * AARCH_PARSE_OK: Handling was sucessful.
-     * AARCH_INVALID_ARG: The type is invalid in this context and the caller
-     should print an error.
-     * AARCH_INVALID_FEATURE: The type is invalid and the handler prints its
-     own error.  */
-  enum aarch_parse_opt_result (*handler)(char*, char*);
+  /* The type can only appear alone, other types should be rejected.  */
+  int alone;
+  /* Function to handle the protection type and set global variables.  */
+  void (*handler)(void);
   /* A list of types that can follow this type in the option string.  */
   const struct aarch_branch_protect_type* subtypes;
   unsigned int num_subtypes;

gcc/config/arm/arm.cc

@@ -2433,8 +2433,6 @@ const struct tune_params arm_fa726te_tune =
   tune_params::SCHED_AUTOPREF_OFF
 };
 
-char *accepted_branch_protection_string = NULL;
-
 /* Auto-generated CPU, FPU and architecture tables.  */
 #include "arm-cpu-data.h"
 
@@ -3308,7 +3306,8 @@ arm_configure_build_target (struct arm_build_target *target,
 
   if (opts->x_arm_branch_protection_string)
     {
-      aarch_validate_mbranch_protection (opts->x_arm_branch_protection_string);
+      aarch_validate_mbranch_protection (opts->x_arm_branch_protection_string,
+					 "-mbranch-protection=");
 
       if (aarch_ra_sign_key != AARCH_KEY_A)
 	{

gcc/config/gcn/gcn-hsa.h

@@ -75,16 +75,15 @@ extern unsigned int gcn_local_sym_hash (const char *name);
    supported for gcn.  */
 #define GOMP_SELF_SPECS ""
 
-#define NO_XNACK "!march=*:;march=fiji:;march=gfx1030:;"
+#define NO_XNACK "march=fiji:;march=gfx1030:;" \
+    /* These match the defaults set in gcn.cc.  */ \
+    "!mxnack*|mxnack=default:%{march=gfx900|march=gfx906|march=gfx908:-mattr=-xnack};"
 #define NO_SRAM_ECC "!march=*:;march=fiji:;march=gfx900:;march=gfx906:;"
 
 /* In HSACOv4 no attribute setting means the binary supports "any" hardware
    configuration.  The name of the attribute also changed.  */
 #define SRAMOPT "msram-ecc=on:-mattr=+sramecc;msram-ecc=off:-mattr=-sramecc"
-
-/* Replace once XNACK is supported:
-   #define XNACKOPT "mxnack=on:-mattr=+xnack;mxnack=off:-mattr=-xnack"  */
-#define XNACKOPT "!mnack=*:-mattr=-xnack;mnack=*:-mattr=-xnack"
+#define XNACKOPT "mxnack=on:-mattr=+xnack;mxnack=off:-mattr=-xnack"
 
 /* Use LLVM assembler and linker options.  */
 #define ASM_SPEC  "-triple=amdgcn--amdhsa "  \

gcc/config/gcn/gcn-opts.h

@@ -65,7 +65,8 @@ enum hsaco_attr_type
 {
   HSACO_ATTR_OFF,
   HSACO_ATTR_ON,
-  HSACO_ATTR_ANY
+  HSACO_ATTR_ANY,
+  HSACO_ATTR_DEFAULT
 };
 
 #endif

gcc/config/gcn/gcn-valu.md

@@ -1145,13 +1145,13 @@
     {})
 
 (define_insn "gather<mode>_insn_1offset<exec>"
-  [(set (match_operand:V_MOV 0 "register_operand"		   "=v,a")
+  [(set (match_operand:V_MOV 0 "register_operand"		   "=v,a,&v,&a")
 	(unspec:V_MOV
-	  [(plus:<VnDI> (match_operand:<VnDI> 1 "register_operand" " v,v")
+	  [(plus:<VnDI> (match_operand:<VnDI> 1 "register_operand" " v,v, v, v")
 			(vec_duplicate:<VnDI>
-			  (match_operand 2 "immediate_operand"	   " n,n")))
-	   (match_operand 3 "immediate_operand"			   " n,n")
-	  (match_operand 4 "immediate_operand"			   " n,n")
+			  (match_operand 2 "immediate_operand"	   " n,n, n, n")))
+	   (match_operand 3 "immediate_operand"			   " n,n, n, n")
+	   (match_operand 4 "immediate_operand"			   " n,n, n, n")
 	   (mem:BLK (scratch))]
 	  UNSPEC_GATHER))]
   "(AS_FLAT_P (INTVAL (operands[3]))
@@ -1182,7 +1182,8 @@
   }
   [(set_attr "type" "flat")
    (set_attr "length" "12")
-   (set_attr "gcn_version" "*,cdna2")])
+   (set_attr "gcn_version" "*,cdna2,*,cdna2")
+   (set_attr "xnack" "off,off,on,on")])
 
 (define_insn "gather<mode>_insn_1offset_ds<exec>"
   [(set (match_operand:V_MOV 0 "register_operand"		   "=v,a")
@@ -1208,18 +1209,18 @@
    (set_attr "gcn_version" "*,cdna2")])
 
 (define_insn "gather<mode>_insn_2offsets<exec>"
-  [(set (match_operand:V_MOV 0 "register_operand"		      "=v,a")
+  [(set (match_operand:V_MOV 0 "register_operand"		"=v,a,&v,&a")
 	(unspec:V_MOV
 	  [(plus:<VnDI>
 	     (plus:<VnDI>
 	       (vec_duplicate:<VnDI>
-		 (match_operand:DI 1 "register_operand"		      "Sv,Sv"))
+		 (match_operand:DI 1 "register_operand"		"Sv,Sv,Sv,Sv"))
 	       (sign_extend:<VnDI>
-		 (match_operand:<VnSI> 2 "register_operand"	      " v,v")))
+		 (match_operand:<VnSI> 2 "register_operand"	" v, v, v, v")))
 	     (vec_duplicate:<VnDI> (match_operand 3 "immediate_operand"
-								      " n,n")))
-	   (match_operand 4 "immediate_operand"			      " n,n")
-	   (match_operand 5 "immediate_operand"			      " n,n")
+								" n, n, n, n")))
+	   (match_operand 4 "immediate_operand"			" n, n, n, n")
+	   (match_operand 5 "immediate_operand"			" n, n, n, n")
 	   (mem:BLK (scratch))]
 	  UNSPEC_GATHER))]
   "(AS_GLOBAL_P (INTVAL (operands[4]))
@@ -1239,7 +1240,8 @@
   }
   [(set_attr "type" "flat")
    (set_attr "length" "12")
-   (set_attr "gcn_version" "*,cdna2")])
+   (set_attr "gcn_version" "*,cdna2,*,cdna2")
+   (set_attr "xnack" "off,off,on,on")])
 
 (define_expand "scatter_store<mode><vnsi>"
   [(match_operand:DI 0 "register_operand")

gcc/config/gcn/gcn.cc

@@ -160,11 +160,41 @@ gcn_option_override (void)
 	acc_lds_size = 32768;
     }
 
-  /* The xnack option is a placeholder, for now.  Before removing, update
-     gcn-hsa.h's XNACKOPT, gcn.opt's mxnack= default init+descr, and
-     invoke.texi's default description.  */
-  if (flag_xnack != HSACO_ATTR_OFF)
-    sorry ("XNACK support");
+  /* gfx803 "Fiji" and gfx1030 do not support XNACK.  */
+  if (gcn_arch == PROCESSOR_FIJI
+      || gcn_arch == PROCESSOR_GFX1030)
+    {
+      if (flag_xnack == HSACO_ATTR_ON)
+	error ("-mxnack=on is incompatible with -march=%s",
+	       (gcn_arch == PROCESSOR_FIJI ? "fiji"
+	        : gcn_arch == PROCESSOR_GFX1030 ? "gfx1030"
+	        : NULL));
+      /* Allow HSACO_ATTR_ANY silently because that's the default.  */
+      flag_xnack = HSACO_ATTR_OFF;
+    }
+
+  /* There's no need for XNACK on devices without USM, and there are register
+     allocation problems caused by the early-clobber when AVGPR spills are not
+     available.
+     FIXME: can the regalloc mean the default can be really "any"?  */
+  if (flag_xnack == HSACO_ATTR_DEFAULT)
+    switch (gcn_arch)
+      {
+      case PROCESSOR_FIJI:
+      case PROCESSOR_VEGA10:
+      case PROCESSOR_VEGA20:
+      case PROCESSOR_GFX908:
+	flag_xnack = HSACO_ATTR_OFF;
+	break;
+      case PROCESSOR_GFX90a:
+	flag_xnack = HSACO_ATTR_ANY;
+	break;
+      default:
+	gcc_unreachable ();
+      }
+
+  if (flag_sram_ecc == HSACO_ATTR_DEFAULT)
+    flag_sram_ecc = HSACO_ATTR_ANY;
 }
 
 /* }}}  */
@@ -3585,18 +3615,20 @@ gcn_expand_epilogue (void)
       /* Assume that an exit value compatible with gcn-run is expected.
          That is, the third input parameter is an int*.
 
-         We can't allocate any new registers, but the kernarg_reg is
-         dead after this, so we'll use that.  */
+         We can't allocate any new registers, but the dispatch_ptr and
+	 kernarg_reg are dead after this, so we'll use those.  */
+      rtx dispatch_ptr_reg = gen_rtx_REG (DImode, cfun->machine->args.reg
+					  [DISPATCH_PTR_ARG]);
       rtx kernarg_reg = gen_rtx_REG (DImode, cfun->machine->args.reg
 				     [KERNARG_SEGMENT_PTR_ARG]);
       rtx retptr_mem = gen_rtx_MEM (DImode,
 				    gen_rtx_PLUS (DImode, kernarg_reg,
 						  GEN_INT (16)));
       set_mem_addr_space (retptr_mem, ADDR_SPACE_SCALAR_FLAT);
-      emit_move_insn (kernarg_reg, retptr_mem);
+      emit_move_insn (dispatch_ptr_reg, retptr_mem);
 
       rtx retval_addr = gen_rtx_REG (DImode, FIRST_VPARM_REG + 2);
-      emit_move_insn (retval_addr, kernarg_reg);
+      emit_move_insn (retval_addr, dispatch_ptr_reg);
       rtx retval_mem = gen_rtx_MEM (SImode, retval_addr);
       set_mem_addr_space (retval_mem, ADDR_SPACE_FLAT);
       emit_move_insn (retval_mem, gen_rtx_REG (SImode, RETURN_VALUE_REG));

gcc/config/gcn/gcn.md

@@ -295,6 +295,8 @@
 (define_attr "gcn_version" "gcn3,gcn5,cdna2" (const_string "gcn3"))
 (define_attr "rdna" "any,no,yes" (const_string "any"))
 
+(define_attr "xnack" "na,off,on" (const_string "na"))
+
 (define_attr "enabled" ""
   (cond [(and (eq_attr "rdna" "no")
 	      (ne (symbol_ref "TARGET_RDNA2") (const_int 0)))
@@ -302,14 +304,19 @@
 	 (and (eq_attr "rdna" "yes")
 	      (eq (symbol_ref "TARGET_RDNA2") (const_int 0)))
 	   (const_int 0)
-	 (eq_attr "gcn_version" "gcn3") (const_int 1)
 	 (and (eq_attr "gcn_version" "gcn5")
-	      (ne (symbol_ref "TARGET_GCN5_PLUS") (const_int 0)))
-	   (const_int 1)
+	      (eq (symbol_ref "TARGET_GCN5_PLUS") (const_int 0)))
+	   (const_int 0)
 	 (and (eq_attr "gcn_version" "cdna2")
-	      (ne (symbol_ref "TARGET_CDNA2_PLUS") (const_int 0)))
-	   (const_int 1)]
-	(const_int 0)))
+	      (eq (symbol_ref "TARGET_CDNA2_PLUS") (const_int 0)))
+	   (const_int 0)
+	 (and (eq_attr "xnack" "off")
+	      (ne (symbol_ref "TARGET_XNACK") (const_int 0)))
+	   (const_int 0)
+	 (and (eq_attr "xnack" "on")
+	      (eq (symbol_ref "TARGET_XNACK") (const_int 0)))
+	   (const_int 0)]
+	(const_int 1)))
 
 ; We need to be able to identify v_readlane and v_writelane with
 ; SGPR lane selection in order to handle "Manually Inserted Wait States".
@@ -508,9 +515,9 @@
 
 (define_insn "*movbi"
   [(set (match_operand:BI 0 "nonimmediate_operand"
-				    "=Sg,   v,Sg,cs,cV,cV,Sm,RS, v,RF, v,RM")
+			  "=Sg,   v,Sg,cs,cV,cV,Sm,&Sm,RS, v,&v,RF, v,&v,RM")
 	(match_operand:BI 1 "gcn_load_operand"
-				    "SSA,vSvA, v,SS, v,SS,RS,Sm,RF, v,RM, v"))]
+			  "SSA,vSvA, v,SS, v,SS,RS, RS,Sm,RF,RF, v,RM,RM, v"))]
   ""
   {
     /* SCC as an operand is currently not accepted by the LLVM assembler, so
@@ -537,25 +544,29 @@
       return "s_mov_b32\tvcc_lo, %1\;"
 	     "s_mov_b32\tvcc_hi, 0";
     case 6:
-      return "s_load_dword\t%0, %A1\;s_waitcnt\tlgkmcnt(0)";
     case 7:
-      return "s_store_dword\t%1, %A0";
+      return "s_load_dword\t%0, %A1\;s_waitcnt\tlgkmcnt(0)";
     case 8:
-      return "flat_load_dword\t%0, %A1%O1%g1\;s_waitcnt\t0";
+      return "s_store_dword\t%1, %A0";
     case 9:
-      return "flat_store_dword\t%A0, %1%O0%g0";
     case 10:
-      return "global_load_dword\t%0, %A1%O1%g1\;s_waitcnt\tvmcnt(0)";
+      return "flat_load_dword\t%0, %A1%O1%g1\;s_waitcnt\t0";
     case 11:
+      return "flat_store_dword\t%A0, %1%O0%g0";
+    case 12:
+    case 13:
+      return "global_load_dword\t%0, %A1%O1%g1\;s_waitcnt\tvmcnt(0)";
+    case 14:
       return "global_store_dword\t%A0, %1%O0%g0";
     default:
       gcc_unreachable ();
     }
   }
-  [(set_attr "type" "sop1,vop1,vop3a,sopk,vopc,mult,smem,smem,flat,flat,
-		     flat,flat")
-   (set_attr "exec" "*,*,none,*,*,*,*,*,*,*,*,*")
-   (set_attr "length" "4,4,4,4,4,8,12,12,12,12,12,12")])
+  [(set_attr "type" "sop1,vop1,vop3a,sopk,vopc,mult,smem,smem,smem,flat,flat,
+		     flat,flat,flat,flat")
+   (set_attr "exec" "*,*,none,*,*,*,*,*,*,*,*,*,*,*,*")
+   (set_attr "length" "4,4,4,4,4,8,12,12,12,12,12,12,12,12,12")
+   (set_attr "xnack" "*,*,*,*,*,*,off,on,*,off,on,*,off,on,*")])
 
 ; 32bit move pattern
 
@@ -563,32 +574,38 @@
   [(set (match_operand:SISF 0 "nonimmediate_operand")
 	(match_operand:SISF 1 "gcn_load_operand"))]
   ""
-  {@ [cons: =0, 1; attrs: type, exec, length, gcn_version]
-   [SD  ,SSA ;sop1 ,*   ,4 ,*    ] s_mov_b32\t%0, %1
-   [SD  ,J   ;sopk ,*   ,4 ,*    ] s_movk_i32\t%0, %1
-   [SD  ,B   ;sop1 ,*   ,8 ,*    ] s_mov_b32\t%0, %1
-   [SD  ,RB  ;smem ,*   ,12,*    ] s_buffer_load%s0\t%0, s[0:3], %1\;s_waitcnt\tlgkmcnt(0)
-   [RB  ,Sm  ;smem ,*   ,12,*    ] s_buffer_store%s1\t%1, s[0:3], %0
-   [Sm  ,RS  ;smem ,*   ,12,*    ] s_load_dword\t%0, %A1\;s_waitcnt\tlgkmcnt(0)
-   [RS  ,Sm  ;smem ,*   ,12,*    ] s_store_dword\t%1, %A0
-   [v   ,v   ;vop1 ,*   ,4 ,*    ] v_mov_b32\t%0, %1
-   [Sg  ,v   ;vop3a,none,8 ,*    ] v_readlane_b32\t%0, %1, 0
-   [v   ,Sv  ;vop3a,none,8 ,*    ] v_writelane_b32\t%0, %1, 0
-   [v   ,^a  ;vop3p_mai,*,8,*    ] v_accvgpr_read_b32\t%0, %1
-   [a   ,v   ;vop3p_mai,*,8,*    ] v_accvgpr_write_b32\t%0, %1
-   [a   ,a   ;vop1 ,*    ,4,cdna2] v_accvgpr_mov_b32\t%0, %1
-   [v   ,RF  ;flat ,*   ,12,*    ] flat_load_dword\t%0, %A1%O1%g1\;s_waitcnt\t0
-   [^a  ,RF  ;flat ,*   ,12,cdna2] ^
-   [RF  ,v   ;flat ,*   ,12,*    ] flat_store_dword\t%A0, %1%O0%g0
-   [RF  ,a   ;flat ,*   ,12,cdna2] ^
-   [v   ,B   ;vop1 ,*   ,8 ,*    ] v_mov_b32\t%0, %1
-   [RLRG,v   ;ds   ,*   ,12,*    ] ds_write_b32\t%A0, %1%O0\;s_waitcnt\tlgkmcnt(0)
-   [v   ,RLRG;ds   ,*   ,12,*    ] ds_read_b32\t%0, %A1%O1\;s_waitcnt\tlgkmcnt(0)
-   [SD  ,Y   ;sop1 ,*   ,8 ,*    ] s_mov_b32\t%0, %1
-   [v   ,RM  ;flat ,*   ,12,*    ] global_load_dword\t%0, %A1%O1%g1\;s_waitcnt\tvmcnt(0)
-   [^a  ,RM  ;flat ,*   ,12,cdna2] ^
-   [RM  ,v   ;flat ,*   ,12,*    ] global_store_dword\t%A0, %1%O0%g0
-   [RM  ,a   ;flat ,*   ,12,cdna2] ^
+  {@ [cons: =0, 1; attrs: type, exec, length, gcn_version, xnack]
+   [SD  ,SSA ;sop1 ,*   ,4 ,*    ,*  ] s_mov_b32\t%0, %1
+   [SD  ,J   ;sopk ,*   ,4 ,*    ,*  ] s_movk_i32\t%0, %1
+   [SD  ,B   ;sop1 ,*   ,8 ,*    ,*  ] s_mov_b32\t%0, %1
+   [SD  ,RB  ;smem ,*   ,12,*    ,off] s_buffer_load%s0\t%0, s[0:3], %1\;s_waitcnt\tlgkmcnt(0)
+   [&SD ,RB  ;smem ,*   ,12,*    ,on ] ^
+   [RB  ,Sm  ;smem ,*   ,12,*    ,*  ] s_buffer_store%s1\t%1, s[0:3], %0
+   [Sm  ,RS  ;smem ,*   ,12,*    ,off] s_load_dword\t%0, %A1\;s_waitcnt\tlgkmcnt(0)
+   [&Sm ,RS  ;smem ,*   ,12,*    ,on ] ^
+   [RS  ,Sm  ;smem ,*   ,12,*    ,*  ] s_store_dword\t%1, %A0
+   [v   ,v   ;vop1 ,*   ,4 ,*    ,*  ] v_mov_b32\t%0, %1
+   [Sg  ,v   ;vop3a,none,8 ,*    ,*  ] v_readlane_b32\t%0, %1, 0
+   [v   ,Sv  ;vop3a,none,8 ,*    ,*  ] v_writelane_b32\t%0, %1, 0
+   [v   ,^a  ;vop3p_mai,*,8,*    ,*  ] v_accvgpr_read_b32\t%0, %1
+   [a   ,v   ;vop3p_mai,*,8,*    ,*  ] v_accvgpr_write_b32\t%0, %1
+   [a   ,a   ;vop1 ,*    ,4,cdna2,*  ] v_accvgpr_mov_b32\t%0, %1
+   [v   ,RF  ;flat ,*   ,12,*    ,off] flat_load_dword\t%0, %A1%O1%g1\;s_waitcnt\t0
+   [&v  ,RF  ;flat ,*   ,12,*    ,on ] ^
+   [^a  ,RF  ;flat ,*   ,12,cdna2,off] ^
+   [&^a ,RF  ;flat ,*   ,12,cdna2,on ] ^
+   [RF  ,v   ;flat ,*   ,12,*    ,*  ] flat_store_dword\t%A0, %1%O0%g0
+   [RF  ,a   ;flat ,*   ,12,cdna2,*  ] ^
+   [v   ,B   ;vop1 ,*   ,8 ,*    ,*  ] v_mov_b32\t%0, %1
+   [RLRG,v   ;ds   ,*   ,12,*    ,*  ] ds_write_b32\t%A0, %1%O0\;s_waitcnt\tlgkmcnt(0)
+   [v   ,RLRG;ds   ,*   ,12,*    ,*  ] ds_read_b32\t%0, %A1%O1\;s_waitcnt\tlgkmcnt(0)
+   [SD  ,Y   ;sop1 ,*   ,8 ,*    ,*  ] s_mov_b32\t%0, %1
+   [v   ,RM  ;flat ,*   ,12,*    ,off] global_load_dword\t%0, %A1%O1%g1\;s_waitcnt\tvmcnt(0)
+   [&v  ,RM  ;flat ,*   ,12,*    ,on ] ^
+   [^a  ,RM  ;flat ,*   ,12,cdna2,off] ^
+   [&^a ,RM  ;flat ,*   ,12,cdna2,on ] ^
+   [RM  ,v   ;flat ,*   ,12,*    ,*  ] global_store_dword\t%A0, %1%O0%g0
+   [RM  ,a   ;flat ,*   ,12,cdna2,*  ] ^
   })
 
 ; 8/16bit move pattern
@@ -598,27 +615,31 @@
   [(set (match_operand:QIHI 0 "nonimmediate_operand")
 	(match_operand:QIHI 1 "gcn_load_operand"))]
   "gcn_valid_move_p (<MODE>mode, operands[0], operands[1])"
-  {@ [cons: =0, 1; attrs: type, exec, length, gcn_version]
-  [SD  ,SSA ;sop1 ,*   ,4 ,*    ] s_mov_b32\t%0, %1
-  [SD  ,J   ;sopk ,*   ,4 ,*    ] s_movk_i32\t%0, %1
-  [SD  ,B   ;sop1 ,*   ,8 ,*    ] s_mov_b32\t%0, %1
-  [v   ,v   ;vop1 ,*   ,4 ,*    ] v_mov_b32\t%0, %1
-  [Sg  ,v   ;vop3a,none,4 ,*    ] v_readlane_b32\t%0, %1, 0
-  [v   ,Sv  ;vop3a,none,4 ,*    ] v_writelane_b32\t%0, %1, 0
-  [v   ,^a  ;vop3p_mai,*,8,*    ] v_accvgpr_read_b32\t%0, %1
-  [a   ,v   ;vop3p_mai,*,8,*    ] v_accvgpr_write_b32\t%0, %1
-  [a   ,a   ;vop1 ,*    ,8,cdna2] v_accvgpr_mov_b32\t%0, %1
-  [v    ,RF ;flat ,*   ,12,*    ] flat_load%o1\t%0, %A1%O1%g1\;s_waitcnt\t0
-  [^a   ,RF ;flat ,*   ,12,cdna2] ^
-  [RF  ,v   ;flat ,*   ,12,*    ] flat_store%s0\t%A0, %1%O0%g0
-  [RF  ,a   ;flat ,*   ,12,cdna2] ^
-  [v   ,B   ;vop1 ,*   ,8 ,*    ] v_mov_b32\t%0, %1
-  [RLRG,v   ;ds   ,*   ,12,*    ] ds_write%b0\t%A0, %1%O0\;s_waitcnt\tlgkmcnt(0)
-  [v   ,RLRG;ds   ,*   ,12,*    ] ds_read%u1\t%0, %A1%O1\;s_waitcnt\tlgkmcnt(0)
-  [v   ,RM  ;flat ,*   ,12,*    ] global_load%o1\t%0, %A1%O1%g1\;s_waitcnt\tvmcnt(0)
-  [^a  ,RM  ;flat ,*   ,12,cdna2] ^
-  [RM  ,v   ;flat ,*   ,12,*    ] global_store%s0\t%A0, %1%O0%g0
-  [RM  ,a   ;flat ,*   ,12,cdna2] ^
+  {@ [cons: =0, 1; attrs: type, exec, length, gcn_version, xnack]
+  [SD  ,SSA ;sop1 ,*   ,4 ,*    ,*  ] s_mov_b32\t%0, %1
+  [SD  ,J   ;sopk ,*   ,4 ,*    ,*  ] s_movk_i32\t%0, %1
+  [SD  ,B   ;sop1 ,*   ,8 ,*    ,*  ] s_mov_b32\t%0, %1
+  [v   ,v   ;vop1 ,*   ,4 ,*    ,*  ] v_mov_b32\t%0, %1
+  [Sg  ,v   ;vop3a,none,4 ,*    ,*  ] v_readlane_b32\t%0, %1, 0
+  [v   ,Sv  ;vop3a,none,4 ,*    ,*  ] v_writelane_b32\t%0, %1, 0
+  [v   ,^a  ;vop3p_mai,*,8,*    ,*  ] v_accvgpr_read_b32\t%0, %1
+  [a   ,v   ;vop3p_mai,*,8,*    ,*  ] v_accvgpr_write_b32\t%0, %1
+  [a   ,a   ;vop1 ,*    ,8,cdna2,*  ] v_accvgpr_mov_b32\t%0, %1
+  [v   ,RF  ;flat ,*   ,12,*    ,off] flat_load%o1\t%0, %A1%O1%g1\;s_waitcnt\t0
+  [&v  ,RF  ;flat ,*   ,12,*    ,on ] ^
+  [^a  ,RF  ;flat ,*   ,12,cdna2,off] ^
+  [&^a ,RF  ;flat ,*   ,12,cdna2,on ] ^
+  [RF  ,v   ;flat ,*   ,12,*    ,*  ] flat_store%s0\t%A0, %1%O0%g0
+  [RF  ,a   ;flat ,*   ,12,cdna2,*  ] ^
+  [v   ,B   ;vop1 ,*   ,8 ,*    ,*  ] v_mov_b32\t%0, %1
+  [RLRG,v   ;ds   ,*   ,12,*    ,*  ] ds_write%b0\t%A0, %1%O0\;s_waitcnt\tlgkmcnt(0)
+  [v   ,RLRG;ds   ,*   ,12,*    ,*  ] ds_read%u1\t%0, %A1%O1\;s_waitcnt\tlgkmcnt(0)
+  [v   ,RM  ;flat ,*   ,12,*    ,off] global_load%o1\t%0, %A1%O1%g1\;s_waitcnt\tvmcnt(0)
+  [&v  ,RM  ;flat ,*   ,12,*    ,on ] ^
+  [^a  ,RM  ;flat ,*   ,12,cdna2,off] ^
+  [&^a ,RM  ;flat ,*   ,12,cdna2,on ] ^
+  [RM  ,v   ;flat ,*   ,12,*    ,*  ] global_store%s0\t%A0, %1%O0%g0
+  [RM  ,a   ;flat ,*   ,12,cdna2,*  ] ^
   })
 
 ; 64bit move pattern
@@ -627,29 +648,34 @@
   [(set (match_operand:DIDF 0 "nonimmediate_operand")
 	(match_operand:DIDF 1 "general_operand"))]
   "GET_CODE(operands[1]) != SYMBOL_REF"
-  {@ [cons: =0, 1; attrs: type, length, gcn_version]
-  [SD  ,SSA ;sop1 ,4 ,*    ] s_mov_b64\t%0, %1
-  [SD  ,C   ;sop1 ,8 ,*    ] ^
-  [SD  ,DB  ;mult ,* ,*    ] #
-  [RS  ,Sm  ;smem ,12,*    ] s_store_dwordx2\t%1, %A0
-  [Sm  ,RS  ;smem ,12,*    ] s_load_dwordx2\t%0, %A1\;s_waitcnt\tlgkmcnt(0)
-  [v   ,v   ;vmult,* ,*    ] #
-  [v   ,DB  ;vmult,* ,*    ] #
-  [Sg  ,v   ;vmult,* ,*    ] #
-  [v   ,Sv  ;vmult,* ,*    ] #
-  [v   ,^a  ;vmult,* ,*    ] #
-  [a   ,v   ;vmult,* ,*    ] #
-  [a   ,a   ;vmult,* ,cdna2] #
-  [v   ,RF  ;flat ,12,*    ] flat_load_dwordx2\t%0, %A1%O1%g1\;s_waitcnt\t0
-  [^a  ,RF  ;flat ,12,cdna2] ^
-  [RF  ,v   ;flat ,12,*    ] flat_store_dwordx2\t%A0, %1%O0%g0
-  [RF  ,a   ;flat ,12,cdna2] ^
-  [RLRG,v   ;ds   ,12,*    ] ds_write_b64\t%A0, %1%O0\;s_waitcnt\tlgkmcnt(0)
-  [v   ,RLRG;ds   ,12,*    ] ds_read_b64\t%0, %A1%O1\;s_waitcnt\tlgkmcnt(0)
-  [v   ,RM  ;flat ,12,*    ] global_load_dwordx2\t%0, %A1%O1%g1\;s_waitcnt\tvmcnt(0)
-  [^a  ,RM  ;flat ,12,cdna2] ^
-  [RM  ,v   ;flat ,12,*    ] global_store_dwordx2\t%A0, %1%O0%g0
-  [RM  ,a   ;flat ,12,cdna2] ^
+  {@ [cons: =0, 1; attrs: type, length, gcn_version, xnack]
+  [SD  ,SSA ;sop1 ,4 ,*    ,*  ] s_mov_b64\t%0, %1
+  [SD  ,C   ;sop1 ,8 ,*    ,*  ] ^
+  [SD  ,DB  ;mult ,* ,*    ,*  ] #
+  [RS  ,Sm  ;smem ,12,*    ,*  ] s_store_dwordx2\t%1, %A0
+  [Sm  ,RS  ;smem ,12,*    ,off] s_load_dwordx2\t%0, %A1\;s_waitcnt\tlgkmcnt(0)
+  [&Sm ,RS  ;smem ,12,*    ,on ] ^
+  [v   ,v   ;vmult,* ,*    ,*  ] #
+  [v   ,DB  ;vmult,* ,*    ,*  ] #
+  [Sg  ,v   ;vmult,* ,*    ,*  ] #
+  [v   ,Sv  ;vmult,* ,*    ,*  ] #
+  [v   ,^a  ;vmult,* ,*    ,*  ] #
+  [a   ,v   ;vmult,* ,*    ,*  ] #
+  [a   ,a   ;vmult,* ,cdna2,*  ] #
+  [v   ,RF  ;flat ,12,*    ,off] flat_load_dwordx2\t%0, %A1%O1%g1\;s_waitcnt\t0
+  [&v  ,RF  ;flat ,12,*    ,on ] ^
+  [^a  ,RF  ;flat ,12,cdna2,off] ^
+  [&^a ,RF  ;flat ,12,cdna2,on ] ^
+  [RF  ,v   ;flat ,12,*    ,*  ] flat_store_dwordx2\t%A0, %1%O0%g0
+  [RF  ,a   ;flat ,12,cdna2,*  ] ^
+  [RLRG,v   ;ds   ,12,*    ,*  ] ds_write_b64\t%A0, %1%O0\;s_waitcnt\tlgkmcnt(0)
+  [v   ,RLRG;ds   ,12,*    ,*  ] ds_read_b64\t%0, %A1%O1\;s_waitcnt\tlgkmcnt(0)
+  [v   ,RM  ;flat ,12,*    ,off] global_load_dwordx2\t%0, %A1%O1%g1\;s_waitcnt\tvmcnt(0)
+  [&v  ,RM  ;flat ,12,*    ,on ] ^
+  [^a  ,RM  ;flat ,12,cdna2,off] ^
+  [&^a ,RM  ;flat ,12,cdna2,on ] ^
+  [RM  ,v   ;flat ,12,*    ,*  ] global_store_dwordx2\t%A0, %1%O0%g0
+  [RM  ,a   ;flat ,12,cdna2,*  ] ^
   }
   "reload_completed
    && ((!MEM_P (operands[0]) && !MEM_P (operands[1])
@@ -687,26 +713,31 @@
   [(set (match_operand:TI 0 "nonimmediate_operand")
 	(match_operand:TI 1 "general_operand"  ))]
   ""
-  {@ [cons: =0, 1; attrs: type, delayeduse, length, gcn_version]
-  [SD,SSB;mult ,*  ,* ,*    ] #
-  [RS,Sm ;smem ,*  ,12,*    ] s_store_dwordx4\t%1, %A0
-  [Sm,RS ;smem ,yes,12,*    ] s_load_dwordx4\t%0, %A1\;s_waitcnt\tlgkmcnt(0)
-  [RF,v  ;flat ,*  ,12,*    ] flat_store_dwordx4\t%A0, %1%O0%g0
-  [RF,a  ;flat ,*  ,12,cdna2] ^
-  [v ,RF ;flat ,*  ,12,*    ] flat_load_dwordx4\t%0, %A1%O1%g1\;s_waitcnt\t0
-  [^a,RF ;flat ,*  ,12,cdna2] ^
-  [v ,v  ;vmult,*  ,* ,*    ] #
-  [v ,Sv ;vmult,*  ,* ,*    ] #
-  [SD,v  ;vmult,*  ,* ,*    ] #
-  [RM,v  ;flat ,yes,12,*    ] global_store_dwordx4\t%A0, %1%O0%g0
-  [RM,a  ;flat ,yes,12,cdna2] ^
-  [v ,RM ;flat ,*  ,12,*    ] global_load_dwordx4\t%0, %A1%O1%g1\;s_waitcnt\tvmcnt(0)
-  [^a,RM ;flat ,*  ,12,cdna2] ^
-  [RL,v  ;ds   ,*  ,12,*    ] ds_write_b128\t%A0, %1%O0\;s_waitcnt\tlgkmcnt(0)
-  [v ,RL ;ds   ,*  ,12,*    ] ds_read_b128\t%0, %A1%O1\;s_waitcnt\tlgkmcnt(0)
-  [v ,^a ;vmult,*  ,* ,*    ] #
-  [a ,v  ;vmult,*  ,* ,*    ] #
-  [a ,a  ;vmult,*  ,* ,cdna2] #
+  {@ [cons: =0, 1; attrs: type, delayeduse, length, gcn_version, xnack]
+  [SD ,SSB;mult ,*  ,* ,*    ,*  ] #
+  [RS ,Sm ;smem ,*  ,12,*    ,*  ] s_store_dwordx4\t%1, %A0
+  [Sm ,RS ;smem ,yes,12,*    ,off] s_load_dwordx4\t%0, %A1\;s_waitcnt\tlgkmcnt(0)
+  [&Sm,RS ;smem ,yes,12,*    ,on ] ^
+  [RF ,v  ;flat ,*  ,12,*    ,*  ] flat_store_dwordx4\t%A0, %1%O0%g0
+  [RF ,a  ;flat ,*  ,12,cdna2,*  ] ^
+  [v  ,RF ;flat ,*  ,12,*    ,off] flat_load_dwordx4\t%0, %A1%O1%g1\;s_waitcnt\t0
+  [&v ,RF ;flat ,*  ,12,*    ,on ] ^
+  [^a ,RF ;flat ,*  ,12,cdna2,off] ^
+  [&^a,RF ;flat ,*  ,12,cdna2,on ] ^
+  [v  ,v  ;vmult,*  ,* ,*    ,*  ] #
+  [v  ,Sv ;vmult,*  ,* ,*    ,*  ] #
+  [SD ,v  ;vmult,*  ,* ,*    ,*  ] #
+  [RM ,v  ;flat ,yes,12,*    ,*  ] global_store_dwordx4\t%A0, %1%O0%g0
+  [RM ,a  ;flat ,yes,12,cdna2,*  ] ^
+  [v  ,RM ;flat ,*  ,12,*    ,off] global_load_dwordx4\t%0, %A1%O1%g1\;s_waitcnt\tvmcnt(0)
+  [&v ,RM ;flat ,*  ,12,*    ,on ] ^
+  [^a ,RM ;flat ,*  ,12,cdna2,off] ^
+  [&^a,RM ;flat ,*  ,12,cdna2,on ] ^
+  [RL ,v  ;ds   ,*  ,12,*    ,*  ] ds_write_b128\t%A0, %1%O0\;s_waitcnt\tlgkmcnt(0)
+  [v  ,RL ;ds   ,*  ,12,*    ,*  ] ds_read_b128\t%0, %A1%O1\;s_waitcnt\tlgkmcnt(0)
+  [v  ,^a ;vmult,*  ,* ,*    ,*  ] #
+  [a  ,v  ;vmult,*  ,* ,*    ,*  ] #
+  [a  ,a  ;vmult,*  ,* ,cdna2,*  ] #
   }
   "reload_completed
    && REG_P (operands[0])
@@ -889,6 +920,8 @@
   (clobber (reg:BI SCC_REG))]
  "GET_CODE (operands[1]) == SYMBOL_REF || GET_CODE (operands[1]) == LABEL_REF"
   {
+    /* This s_load may not be XNACK-safe on devices where the GOT may fault.
+       DGPUs are most likely fine.  */
     if (SYMBOL_REF_P (operands[1])
 	&& SYMBOL_REF_WEAK (operands[1]))
 	return "s_getpc_b64\t%0\;"
@@ -913,6 +946,8 @@
   {
     /* !!! These sequences clobber CC_SAVE_REG.  */
 
+    /* This s_load may not be XNACK-safe on devices where the GOT may fault.
+       DGPUs are most likely fine.  */
     if (SYMBOL_REF_P (operands[1])
 	&& SYMBOL_REF_WEAK (operands[1]))
 	return "s_mov_b32\ts22, scc\;"

gcc/config/gcn/gcn.opt

@@ -97,9 +97,12 @@ Enum(hsaco_attr_type) String(on) Value(HSACO_ATTR_ON)
 EnumValue
 Enum(hsaco_attr_type) String(any) Value(HSACO_ATTR_ANY)
 
+EnumValue
+Enum(hsaco_attr_type) String(default) Value(HSACO_ATTR_DEFAULT)
+
 mxnack=
-Target RejectNegative Joined ToLower Enum(hsaco_attr_type) Var(flag_xnack) Init(HSACO_ATTR_OFF)
-Compile for devices requiring XNACK enabled. Default \"off\".
+Target RejectNegative Joined ToLower Enum(hsaco_attr_type) Var(flag_xnack) Init(HSACO_ATTR_DEFAULT)
+Compile for devices requiring XNACK enabled. Default \"any\" if USM is supported.
 
 msram-ecc=
 Target RejectNegative Joined ToLower Enum(hsaco_attr_type) Var(flag_sram_ecc) Init(HSACO_ATTR_ANY)

gcc/config/h8300/addsub.md

@@ -239,3 +239,80 @@
   "reload_completed"
   "xor.w\\t#32768,%e0"
   [(set_attr "length" "4")])
+
+(define_expand "uaddv<mode>4"
+  [(set (match_operand:QHSI 0 "register_operand" "")
+	(plus:QHSI (match_operand:QHSI 1 "register_operand" "")
+		   (match_operand:QHSI 2 "register_operand" "")))
+   (set (pc)
+	(if_then_else (ltu (match_dup 0) (match_dup 1))
+		      (label_ref (match_operand 3 ""))
+		      (pc)))]
+  "")
+
+(define_insn_and_split "*uaddv"
+  [(set (match_operand:QHSI2 3 "register_operand" "=&r")
+	(ltu:QHSI2 (plus:QHSI (match_operand:QHSI 1 "register_operand" "%0")
+			   (match_operand:QHSI 2 "register_operand" "r"))
+		(match_dup 1)))
+   (set (match_operand:QHSI 0 "register_operand" "=r")
+	(plus:QHSI (match_dup 1) (match_dup 2)))]
+  ""
+  "#"
+  "&& reload_completed"
+  [(parallel [(set (match_dup 3) (ltu:QHSI2 (plus:QHSI (match_dup 1) (match_dup 2))
+					  (match_dup 1)))
+	      (set (match_dup 0) (plus:QHSI (match_dup 1) (match_dup 2)))
+	      (clobber (reg:CC CC_REG))])])
+
+(define_insn "*uaddv"
+  [(set (match_operand:QHSI2 3 "register_operand" "=&r")
+	(ltu:QHSI2 (plus:QHSI (match_operand:QHSI 1 "register_operand" "%0")
+			     (match_operand:QHSI 2 "register_operand" "r"))
+		(match_dup 1)))
+   (set (match_operand:QHSI 0 "register_operand" "=r")
+	(plus (match_dup 1) (match_dup 2)))
+   (clobber (reg:CC CC_REG))]
+  ""
+{
+  if (E_<QHSI2:MODE>mode == E_QImode)
+    {
+      if (E_<QHSI:MODE>mode == E_QImode)
+	return "sub.b\t%X3,%X3\;add.b\t%X2,%X0\;addx\t%X3,%X3";
+      else if (E_<QHSI:MODE>mode == E_HImode)
+	return "sub.b\t%X3,%X3\;add.w\t%T2,%T0\;addx\t%X3,%X3";
+      else if (E_<QHSI:MODE>mode == E_SImode)
+	return "sub.b\t%X3,%X3\;add.l\t%S2,%S0\;addx\t%X3,%X3";
+    }
+  else if (E_<QHSI2:MODE>mode == E_HImode)
+    {
+      if (E_<QHSI:MODE>mode == E_QImode)
+	return "sub.w\t%T3,%T3\;add.b\t%X2,%X0\;addx\t%X3,%X3";
+      else if (E_<QHSI:MODE>mode == E_HImode)
+	return "sub.w\t%T3,%T3\;add.w\t%T2,%T0\;addx\t%X3,%X3";
+      else if (E_<QHSI:MODE>mode == E_SImode)
+	return "sub.w\t%T3,%T3\;add.l\t%S2,%S0\;addx\t%X3,%X3";
+    }
+  else if (E_<QHSI2:MODE>mode == E_SImode)
+    {
+      if (E_<QHSI:MODE>mode == E_QImode)
+	return "sub.l\t%S3,%S3\;add.b\t%X2,%X0\;addx\t%X3,%X3";
+      else if (E_<QHSI:MODE>mode == E_HImode)
+	return "sub.l\t%S3,%S3\;add.w\t%T2,%T0\;addx\t%X3,%X3";
+      else if (E_<QHSI:MODE>mode == E_SImode)
+	return "sub.l\t%S3,%S3\;add.l\t%S2,%S0\;addx\t%X3,%X3";
+    }
+  else
+    gcc_unreachable ();
+}
+  [(set_attr "length" "6")])
+
+(define_expand "usubv<mode>4"
+  [(set (match_operand:QHSI 0 "register_operand" "")
+	(minus:QHSI (match_operand:QHSI 1 "register_operand" "")
+		    (match_operand:QHSI 2 "register_operand" "")))
+   (set (pc)
+	(if_then_else (ltu (match_dup 1) (match_dup 2))
+		      (label_ref (match_operand 3 ""))
+		      (pc)))]
+  "")

gcc/config/h8300/combiner.md

@@ -1269,7 +1269,54 @@
 ;; 		      (pc)))]
 ;;   "")
 
-;; Various ways to extract a single bit bitfield and sign extend it
+;; This is a signed bitfield extraction starting at bit 0
+;; It's usually faster than using shifts, but not always,
+;; particularly on the H8/S and H8/SX variants.
+(define_insn_and_split "*extvsi_n_0"
+  [(set (match_operand:SI 0 "register_operand" "=r")
+	(sign_extract:SI (match_operand:SI 1 "register_operand" "0")
+			 (match_operand 2 "const_int_operand")
+			 (const_int 0)))]
+  "INTVAL (operands[2]) > 1
+   && INTVAL (operands[2]) < (TARGET_H8300S ? 26 - TARGET_H8300SX : 29)
+   && (!TARGET_H8300SX || (INTVAL (operands[2]) != 24 && INTVAL (operands[2]) != 17))"
+  "#"
+  "&& reload_completed"
+[(parallel [(set (match_dup 0) (and:SI (match_dup 0) (match_dup 3)))
+	    (clobber (reg:CC CC_REG))])
+ (parallel [(set (match_dup 0) (xor:SI (match_dup 0) (match_dup 4)))
+	    (clobber (reg:CC CC_REG))])
+ (parallel [(set (match_dup 0) (minus:SI (match_dup 0) (match_dup 4)))
+	    (clobber (reg:CC CC_REG))])]
+{
+  int tmp = INTVAL (operands[2]);
+  operands[3] = GEN_INT (~(HOST_WIDE_INT_M1U << tmp));
+  operands[4] = GEN_INT (HOST_WIDE_INT_1U << (tmp - 1));
+})
+
+(define_insn_and_split "*extvsi_n_n"
+  [(set (match_operand:SI 0 "register_operand" "=r")
+	(sign_extract:SI (match_operand:SI 1 "register_operand" "0")
+			 (match_operand 2 "const_int_operand")
+			 (match_operand 3 "const_int_operand")))]
+  "(!h8300_shift_needs_scratch_p (INTVAL (operands[3]), SImode, LSHIFTRT)
+    && use_extvsi (INTVAL (operands[2]), INTVAL (operands[3])))"
+  "#"
+  "&& reload_completed"
+[(parallel [(set (match_dup 0) (lshiftrt:SI (match_dup 0) (match_dup 3)))
+	    (clobber (reg:CC CC_REG))])
+ (parallel [(set (match_dup 0) (and:SI (match_dup 0) (match_dup 4)))
+	    (clobber (reg:CC CC_REG))])
+ (parallel [(set (match_dup 0) (xor:SI (match_dup 0) (match_dup 5)))
+	    (clobber (reg:CC CC_REG))])
+ (parallel [(set (match_dup 0) (minus:SI (match_dup 0) (match_dup 5)))
+	    (clobber (reg:CC CC_REG))])]
+{
+  int tmp = INTVAL (operands[2]);
+  operands[4] = gen_int_mode (~(HOST_WIDE_INT_M1U << tmp), SImode);
+  operands[5] = gen_int_mode (HOST_WIDE_INT_1U << (tmp - 1), SImode);
+})
+
 ;;
 ;; Testing showed this only triggering with SImode, probably because
 ;; of how insv/extv are defined.
@@ -1358,7 +1405,7 @@
      to get that bit into the destination, properly extended.  */
   return "subx\t%s0,%s0\;exts.w %T0\;exts.l %0";
 }
-  [(set_attr "length" "10")])
+  [(set (attr "length") (symbol_ref "INTVAL (operands[2]) >= 16 ? 10 : 8"))])
 
 ;; For shift counts >= 16 we can always do better than the
 ;; generic sequences.  Other patterns handle smaller counts.

gcc/config/h8300/h8300-protos.h

@@ -111,5 +111,6 @@ extern const char *    output_h8sx_shift (rtx *, int, int);
 extern bool            h8300_operands_match_p (rtx *);
 extern bool            h8sx_mergeable_memrefs_p (rtx, rtx);
 extern poly_int64      h8300_push_rounding (poly_int64);
+extern bool            use_extvsi (int, int);
 
 #endif /* ! GCC_H8300_PROTOS_H */

gcc/config/h8300/h8300.cc

@@ -4299,6 +4299,11 @@ compute_a_shift_length (rtx operands[3], rtx_code code)
 	  /* Fall through.  */
 
 	case SHIFT_INLINE:
+	  /* H8/SX has a richer set of logical shifts.  */
+	  if (TARGET_H8300SX
+	      && (code == ASHIFT || code == LSHIFTRT))
+	    return (exact_log2 (n) >= 0) ? 2 : 4;
+
 	  n = info.remainder;
 
 	  if (info.shift2 != NULL)
@@ -5498,6 +5503,70 @@ h8300_trampoline_init (rtx m_tramp, tree fndecl, rtx cxt)
     }
 }
 
+/* Return true if a signed bitfield extraction with length COUNT
+   starting at position POS should be optimized by first shifting
+   right to put the field in the LSB, then using a 3 operand sequence
+   to sign extend from an arbitrary position.  Return false
+   otherwise.
+
+   The basic idea here is to compute the length of each sequence
+   and use that as a proxy for performance.  It's not strictly
+   correct on the H8/SX which has variable timed shifts and some
+   lengths may be incorrect, but this is pretty close.
+
+   There may be cases where the length computations are inaccurate
+   which may in turn lead to a sub-optimal sequence, but that
+   should be rare.
+
+   We do not try to balance avoiding a loop with burning an extra
+   couple bytes.   Those probably couple be handled with special
+   cases.  */
+
+bool
+use_extvsi (int count, int pos)
+{
+  rtx operands[3];
+  operands[0] = gen_rtx_REG (SImode, 0);
+  operands[1] = gen_rtx_REG (SImode, 0);
+
+  /* We have a special sequence to sign extract a single bit
+     object, otherwise compute it as a pair of shifts, first
+     left, then arithmetic right.  The cost of that special
+     sequence is 8/10 depending on where the bit is.  */
+  unsigned shift_cost;
+  if (count == 1)
+    shift_cost = pos >= 16 ? 10 : 8;
+  else
+    {
+      unsigned lshift = 32 - (count + pos);
+      unsigned rshift = 32 - count;
+      operands[2] = GEN_INT (lshift);
+      shift_cost = compute_a_shift_length (operands, ASHIFT);
+      operands[2] = GEN_INT (rshift);
+      shift_cost += compute_a_shift_length (operands, ASHIFTRT);
+    }
+
+  /* Cost of hopefully optimized sequence.  First we logically shift right
+     by an adjusted count.  Logicals are generally better than arith,
+     particularly for H8/SX.  */
+  operands[2] = GEN_INT (pos);
+  unsigned opt_cost = compute_a_shift_length (operands, LSHIFTRT);
+  operands[2] = gen_int_mode (~(HOST_WIDE_INT_M1U << count), SImode);
+  opt_cost += compute_logical_op_length (SImode, AND, operands, NULL);
+  operands[2] = gen_int_mode (HOST_WIDE_INT_1U << (count - 1), SImode);
+  opt_cost += compute_logical_op_length (SImode, XOR, operands, NULL);
+
+  /* H8/SX has short form subtraction.  */
+  if (TARGET_H8300SX && (INTVAL (operands[2]) >= 1 && INTVAL (operands[2]) <= 7))
+    opt_cost += 2;
+  else if (TARGET_H8300SX && (INTVAL (operands[2]) >= 8 && INTVAL (operands[2]) <= 32767))
+    opt_cost += 4;
+  else
+    opt_cost += 6;
+
+  return opt_cost <= shift_cost;
+}
+
 /* Implement PUSH_ROUNDING.
 
    On the H8/300, @-sp really pushes a byte if you ask it to - but that's

gcc/config/i386/constraints.md

@@ -433,3 +433,8 @@
 
 (define_register_constraint  "jc"
  "TARGET_APX_EGPR && !TARGET_AVX ? GENERAL_GPR16 : GENERAL_REGS")
+
+(define_constraint  "je"
+  "@internal constant that do not allow any unspec global offsets"
+  (and (match_operand 0 "x86_64_immediate_operand")
+       (match_test "!x86_poff_operand_p (op)")))

gcc/config/i386/driver-i386.cc

@@ -897,8 +897,13 @@ const char *host_detect_local_cpu (int argc, const char **argv)
 	      }
 	    /* Never push -mno-avx10.1-{256,512} under -march=native to
 	       avoid unnecessary warnings when building librarys.  */
-	    else if ((isa_names_table[i].feature != FEATURE_AVX10_1_256)
-		     && (isa_names_table[i].feature != FEATURE_AVX10_1_512)
+	    else if (isa_names_table[i].feature != FEATURE_AVX10_1_256
+		     && isa_names_table[i].feature != FEATURE_AVX10_1_512
+		     && isa_names_table[i].feature != FEATURE_AVX512PF
+		     && isa_names_table[i].feature != FEATURE_AVX512ER
+		     && isa_names_table[i].feature != FEATURE_AVX5124FMAPS
+		     && isa_names_table[i].feature != FEATURE_AVX5124VNNIW
+		     && isa_names_table[i].feature != FEATURE_PREFETCHWT1
 		     && check_avx512_features (cpu_model, cpu_features2,
 					       isa_names_table[i].feature))
 	      options = concat (options, neg_option,

gcc/config/i386/i386-expand.cc

@@ -297,6 +297,12 @@ ix86_convert_const_wide_int_to_broadcast (machine_mode mode, rtx op)
   if (!TARGET_INTER_UNIT_MOVES_TO_VEC)
     return nullptr;
 
+  unsigned int msize = GET_MODE_SIZE (mode);
+
+  /* Only optimized for vpbroadcast[bwsd]/vbroadcastss with xmm/ymm/zmm.  */
+  if (msize != 16 && msize != 32 && msize != 64)
+    return nullptr;
+
   /* Convert CONST_WIDE_INT to a non-standard SSE constant integer
      broadcast only if vector broadcast is available.  */
   if (!TARGET_AVX
@@ -309,18 +315,23 @@ ix86_convert_const_wide_int_to_broadcast (machine_mode mode, rtx op)
   HOST_WIDE_INT val = CONST_WIDE_INT_ELT (op, 0);
   HOST_WIDE_INT val_broadcast;
   scalar_int_mode broadcast_mode;
-  if (TARGET_AVX2
+  /* vpbroadcastb zmm requires TARGET_AVX512BW.  */
+  if ((msize == 64 ? TARGET_AVX512BW : TARGET_AVX2)
       && ix86_broadcast (val, GET_MODE_BITSIZE (QImode),
 			 val_broadcast))
     broadcast_mode = QImode;
-  else if (TARGET_AVX2
+  else if ((msize == 64 ? TARGET_AVX512BW : TARGET_AVX2)
 	   && ix86_broadcast (val, GET_MODE_BITSIZE (HImode),
 			      val_broadcast))
     broadcast_mode = HImode;
-  else if (ix86_broadcast (val, GET_MODE_BITSIZE (SImode),
+  /* vbroadcasts[sd] only support memory operand w/o AVX2.
+     When msize == 16, pshufs is used for vec_duplicate.
+     when msize == 64, vpbroadcastd is used, and TARGET_AVX512F must be existed.  */
+  else if ((msize != 32 || TARGET_AVX2)
+	   && ix86_broadcast (val, GET_MODE_BITSIZE (SImode),
 			   val_broadcast))
     broadcast_mode = SImode;
-  else if (TARGET_64BIT
+  else if (TARGET_64BIT && (msize != 32 || TARGET_AVX2)
 	   && ix86_broadcast (val, GET_MODE_BITSIZE (DImode),
 			      val_broadcast))
     broadcast_mode = DImode;
@@ -596,23 +607,17 @@ ix86_broadcast_from_constant (machine_mode mode, rtx op)
       && INTEGRAL_MODE_P (mode))
     return nullptr;
 
+  unsigned int msize = GET_MODE_SIZE (mode);
+  unsigned int inner_size = GET_MODE_SIZE (GET_MODE_INNER ((mode)));
+
   /* Convert CONST_VECTOR to a non-standard SSE constant integer
      broadcast only if vector broadcast is available.  */
-  if (!(TARGET_AVX2
-	|| (TARGET_AVX
-	    && (GET_MODE_INNER (mode) == SImode
-		|| GET_MODE_INNER (mode) == DImode))
-	|| FLOAT_MODE_P (mode))
-      || standard_sse_constant_p (op, mode))
+  if (standard_sse_constant_p (op, mode))
     return nullptr;
 
-  /* Don't broadcast from a 64-bit integer constant in 32-bit mode.
-     We can still put 64-bit integer constant in memory when
-     avx512 embed broadcast is available.  */
-  if (GET_MODE_INNER (mode) == DImode && !TARGET_64BIT
-      && (!TARGET_AVX512F
-	  || (GET_MODE_SIZE (mode) == 64 && !TARGET_EVEX512)
-	  || (GET_MODE_SIZE (mode) < 64 && !TARGET_AVX512VL)))
+  /* vpbroadcast[b,w] is available under TARGET_AVX2.
+     or TARGET_AVX512BW for zmm.  */
+  if (inner_size < 4 && !(msize == 64 ? TARGET_AVX512BW : TARGET_AVX2))
     return nullptr;
 
   if (GET_MODE_INNER (mode) == TImode)
@@ -710,7 +715,14 @@ ix86_expand_vector_move (machine_mode mode, rtx operands[])
 	     constant or scalar mem.  */
 	  op1 = gen_reg_rtx (mode);
 	  if (FLOAT_MODE_P (mode)
-	      || (!TARGET_64BIT && GET_MODE_INNER (mode) == DImode))
+	      || (!TARGET_64BIT && GET_MODE_INNER (mode) == DImode)
+	      /* vbroadcastss/vbroadcastsd only supports memory operand
+		 w/o AVX2, force them into memory to avoid spill to
+		 memory.  */
+	      || (GET_MODE_SIZE (mode) == 32
+		  && (GET_MODE_INNER (mode) == DImode
+		      || GET_MODE_INNER (mode) == SImode)
+		  && !TARGET_AVX2))
 	    first = force_const_mem (GET_MODE_INNER (mode), first);
 	  bool ok = ix86_expand_vector_init_duplicate (false, mode,
 						       op1, first);
@@ -1260,14 +1272,14 @@ ix86_swap_binary_operands_p (enum rtx_code code, machine_mode mode,
   return false;
 }
 
-
 /* Fix up OPERANDS to satisfy ix86_binary_operator_ok.  Return the
    destination to use for the operation.  If different from the true
-   destination in operands[0], a copy operation will be required.  */
+   destination in operands[0], a copy operation will be required except
+   under TARGET_APX_NDD.  */
 
 rtx
 ix86_fixup_binary_operands (enum rtx_code code, machine_mode mode,
-			    rtx operands[])
+			    rtx operands[], bool use_ndd)
 {
   rtx dst = operands[0];
   rtx src1 = operands[1];
@@ -1307,7 +1319,7 @@ ix86_fixup_binary_operands (enum rtx_code code, machine_mode mode,
     src1 = force_reg (mode, src1);
 
   /* Source 1 cannot be a non-matching memory.  */
-  if (MEM_P (src1) && !rtx_equal_p (dst, src1))
+  if (!use_ndd && MEM_P (src1) && !rtx_equal_p (dst, src1))
     src1 = force_reg (mode, src1);
 
   /* Improve address combine.  */
@@ -1326,9 +1338,10 @@ ix86_fixup_binary_operands (enum rtx_code code, machine_mode mode,
 
 void
 ix86_fixup_binary_operands_no_copy (enum rtx_code code,
-				    machine_mode mode, rtx operands[])
+				    machine_mode mode, rtx operands[],
+				    bool use_ndd)
 {
-  rtx dst = ix86_fixup_binary_operands (code, mode, operands);
+  rtx dst = ix86_fixup_binary_operands (code, mode, operands, use_ndd);
   gcc_assert (dst == operands[0]);
 }
 
@@ -1338,11 +1351,11 @@ ix86_fixup_binary_operands_no_copy (enum rtx_code code,
 
 void
 ix86_expand_binary_operator (enum rtx_code code, machine_mode mode,
-			     rtx operands[])
+			     rtx operands[], bool use_ndd)
 {
   rtx src1, src2, dst, op, clob;
 
-  dst = ix86_fixup_binary_operands (code, mode, operands);
+  dst = ix86_fixup_binary_operands (code, mode, operands, use_ndd);
   src1 = operands[1];
   src2 = operands[2];
 
@@ -1352,7 +1365,8 @@ ix86_expand_binary_operator (enum rtx_code code, machine_mode mode,
 
   if (reload_completed
       && code == PLUS
-      && !rtx_equal_p (dst, src1))
+      && !rtx_equal_p (dst, src1)
+      && !use_ndd)
     {
       /* This is going to be an LEA; avoid splitting it later.  */
       emit_insn (op);
@@ -1451,7 +1465,7 @@ ix86_expand_vector_logical_operator (enum rtx_code code, machine_mode mode,
 
 bool
 ix86_binary_operator_ok (enum rtx_code code, machine_mode mode,
-			 rtx operands[3])
+			 rtx operands[3], bool use_ndd)
 {
   rtx dst = operands[0];
   rtx src1 = operands[1];
@@ -1475,7 +1489,7 @@ ix86_binary_operator_ok (enum rtx_code code, machine_mode mode,
     return false;
 
   /* Source 1 cannot be a non-matching memory.  */
-  if (MEM_P (src1) && !rtx_equal_p (dst, src1))
+  if (!use_ndd && MEM_P (src1) && !rtx_equal_p (dst, src1))
     /* Support "andhi/andsi/anddi" as a zero-extending move.  */
     return (code == AND
 	    && (mode == HImode
@@ -1492,7 +1506,7 @@ ix86_binary_operator_ok (enum rtx_code code, machine_mode mode,
 
 void
 ix86_expand_unary_operator (enum rtx_code code, machine_mode mode,
-			    rtx operands[])
+			    rtx operands[], bool use_ndd)
 {
   bool matching_memory = false;
   rtx src, dst, op, clob;
@@ -1511,7 +1525,7 @@ ix86_expand_unary_operator (enum rtx_code code, machine_mode mode,
     }
 
   /* When source operand is memory, destination must match.  */
-  if (MEM_P (src) && !matching_memory)
+  if (!use_ndd && MEM_P (src) && !matching_memory)
     src = force_reg (mode, src);
 
   /* Emit the instruction.  */
@@ -6676,6 +6690,142 @@ ix86_split_lshr (rtx *operands, rtx scratch, machine_mode mode)
     }
 }
 
+/* Helper function to split TImode ashl under NDD.  */
+void
+ix86_split_ashl_ndd (rtx *operands, rtx scratch)
+{
+  gcc_assert (TARGET_APX_NDD);
+  int half_width = GET_MODE_BITSIZE (TImode) >> 1;
+
+  rtx low[2], high[2];
+  int count;
+
+  split_double_mode (TImode, operands, 2, low, high);
+  if (CONST_INT_P (operands[2]))
+    {
+      count = INTVAL (operands[2]) & (GET_MODE_BITSIZE (TImode) - 1);
+
+      if (count >= half_width)
+	{
+	  count = count - half_width;
+	  if (count == 0)
+	    {
+	      if (!rtx_equal_p (high[0], low[1]))
+		emit_move_insn (high[0], low[1]);
+	    }
+	  else if (count == 1)
+	    emit_insn (gen_adddi3 (high[0], low[1], low[1]));
+	  else
+	    emit_insn (gen_ashldi3 (high[0], low[1], GEN_INT (count)));
+
+	  ix86_expand_clear (low[0]);
+	}
+      else if (count == 1)
+	{
+	  rtx x3 = gen_rtx_REG (CCCmode, FLAGS_REG);
+	  rtx x4 = gen_rtx_LTU (TImode, x3, const0_rtx);
+	  emit_insn (gen_add3_cc_overflow_1 (DImode, low[0],
+					     low[1], low[1]));
+	  emit_insn (gen_add3_carry (DImode, high[0], high[1], high[1],
+				     x3, x4));
+	}
+      else
+	{
+	  emit_insn (gen_x86_64_shld_ndd (high[0], high[1], low[1],
+					  GEN_INT (count)));
+	  emit_insn (gen_ashldi3 (low[0], low[1], GEN_INT (count)));
+	}
+    }
+  else
+    {
+      emit_insn (gen_x86_64_shld_ndd (high[0], high[1], low[1],
+				      operands[2]));
+      emit_insn (gen_ashldi3 (low[0], low[1], operands[2]));
+      if (TARGET_CMOVE && scratch)
+	{
+	  ix86_expand_clear (scratch);
+	  emit_insn (gen_x86_shift_adj_1
+		     (DImode, high[0], low[0], operands[2], scratch));
+	}
+      else
+	emit_insn (gen_x86_shift_adj_2 (DImode, high[0], low[0], operands[2]));
+    }
+}
+
+/* Helper function to split TImode l/ashr under NDD.  */
+void
+ix86_split_rshift_ndd (enum rtx_code code, rtx *operands, rtx scratch)
+{
+  gcc_assert (TARGET_APX_NDD);
+  int half_width = GET_MODE_BITSIZE (TImode) >> 1;
+  bool ashr_p = code == ASHIFTRT;
+  rtx (*gen_shr)(rtx, rtx, rtx) = ashr_p ? gen_ashrdi3
+					 : gen_lshrdi3;
+
+  rtx low[2], high[2];
+  int count;
+
+  split_double_mode (TImode, operands, 2, low, high);
+  if (CONST_INT_P (operands[2]))
+    {
+      count = INTVAL (operands[2]) & (GET_MODE_BITSIZE (TImode) - 1);
+
+      if (ashr_p && (count == GET_MODE_BITSIZE (TImode) - 1))
+	{
+	  emit_insn (gen_shr (high[0], high[1],
+			      GEN_INT (half_width - 1)));
+	  emit_move_insn (low[0], high[0]);
+	}
+      else if (count >= half_width)
+	{
+	  if (ashr_p)
+	    emit_insn (gen_shr (high[0], high[1],
+				GEN_INT (half_width - 1)));
+	  else
+	    ix86_expand_clear (high[0]);
+
+	  if (count > half_width)
+	    emit_insn (gen_shr (low[0], high[1],
+				GEN_INT (count - half_width)));
+	  else
+	    emit_move_insn (low[0], high[1]);
+	}
+      else
+	{
+	  emit_insn (gen_x86_64_shrd_ndd (low[0], low[1], high[1],
+					  GEN_INT (count)));
+	  emit_insn (gen_shr (high[0], high[1], GEN_INT (count)));
+	}
+    }
+  else
+    {
+      emit_insn (gen_x86_64_shrd_ndd (low[0], low[1], high[1],
+				      operands[2]));
+      emit_insn (gen_shr (high[0], high[1], operands[2]));
+
+      if (TARGET_CMOVE && scratch)
+	{
+	  if (ashr_p)
+	    {
+	      emit_move_insn (scratch, high[0]);
+	      emit_insn (gen_shr (scratch, scratch,
+				  GEN_INT (half_width - 1)));
+	    }
+	  else
+	    ix86_expand_clear (scratch);
+
+	  emit_insn (gen_x86_shift_adj_1
+		     (DImode, low[0], high[0], operands[2], scratch));
+	}
+      else if (ashr_p)
+	emit_insn (gen_x86_shift_adj_3
+		   (DImode, low[0], high[0], operands[2]));
+      else
+	emit_insn (gen_x86_shift_adj_2
+		   (DImode, low[0], high[0], operands[2]));
+    }
+}
+
 /* Expand move of V1TI mode register X to a new TI mode register.  */
 static rtx
 ix86_expand_v1ti_to_ti (rtx x)