Remove per byte precision message truncation for multibyte strings (#…

…1185)

gpdb does not truncate error log correctly on error in external web tables.

If error code is not zero and error message is truncated in interpretError
on multi-byte character, then assert occurs.

If error code is zero and command emit stderr and this error message is
truncated in write_log (which is called by read_err_msg) on multi-byte
character, then bad character writes to log and later this log can not be
correctly read.

Solution is to correct error log truncation according to database encoding.

(cherry picked from commit ae9e7ac)

src/backend/access/external/url_execute.c

@@ -484,7 +484,12 @@ interpretError(int rc, char *buf, size_t buflen, char *err, size_t errlen)
 			/* Exit codes from commands rarely map to strerror() strings. In here
 			 * we show the error string returned from pclose, and omit the non
 			 * friendly exit code interpretation */
-			snprintf(buf, buflen, "error. %s", err);
+			int len = snprintf(buf, buflen, "error. %s", err);
+
+			if (len >= buflen)
+			{
+				buf[pg_mbcliplen(buf, len, buflen - 1)] = '\0';
+			}
 		}
 	}
 	else if (WIFSIGNALED(rc))

src/backend/cdb/cdbthreadlog.c

@@ -117,7 +117,12 @@ write_log(const char *fmt,...)
 	{
 		char		errbuf[2048];	/* Arbitrary size? */
 
-		vsnprintf(errbuf, sizeof(errbuf), fmt, ap);
+		int len = vsnprintf(errbuf, sizeof(errbuf), fmt, ap);
+
+		if (len >= sizeof(errbuf))
+		{
+			errbuf[pg_mbcliplen(errbuf, len, sizeof(errbuf) - 1)] = '\0';
+		}
 
 		/* Write the message in the CSV format */
 		write_message_to_server_log(LOG,

src/test/regress/input/external_table.source

@@ -135,6 +135,15 @@ FETCH FORWARD 1 FROM _psql_cursor;
 CLOSE _psql_cursor;
 COMMIT;
 
+-- ensure correct truncate error log according to database encoding
+CREATE EXTERNAL WEB TABLE table_test (text text) EXECUTE $$ (echo -n ‘ && seq 1 500 | cut -b1 | tr -d '\n' && echo -n ’) >/dev/stderr && false $$ ON ALL FORMAT 'CSV';
+SELECT true FROM table_test;
+DROP EXTERNAL TABLE IF EXISTS table_test;
+CREATE EXTERNAL WEB TABLE table_test (text text) EXECUTE $$ (echo -n ‘ && seq 1 2006 | cut -b1 | tr -d '\n' && echo -n ’) >/dev/stderr && true $$ ON ALL FORMAT 'CSV';
+SELECT true FROM table_test;
+SELECT logseverity FROM gp_toolkit.gp_log_system WHERE logmessage LIKE 'read err msg from pipe%' AND logdatabase = current_database() LIMIT 1;
+DROP EXTERNAL TABLE IF EXISTS table_test;
+
 -- echo will behave differently on different platforms, force to use bash with -E option
 CREATE EXTERNAL WEB TABLE table_qry (val TEXT)
   EXECUTE E'/usr/bin/env bash -c ''echo -E "$GP_QUERY_STRING"''' ON SEGMENT 0

src/test/regress/output/external_table.source

@@ -166,6 +166,25 @@ FETCH FORWARD 1 FROM _psql_cursor;
 
 CLOSE _psql_cursor;
 COMMIT;
+-- ensure correct truncate error log according to database encoding
+CREATE EXTERNAL WEB TABLE table_test (text text) EXECUTE $$ (echo -n ‘ && seq 1 500 | cut -b1 | tr -d '\n' && echo -n ’) >/dev/stderr && false $$ ON ALL FORMAT 'CSV';
+SELECT true FROM table_test;
+ERROR:  external table table_test command ended with error. ‘12345678911111111112222222222333333333344444444445555555555666666666677777777778888888888999999999911111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111112222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333344444444444444444444444444444444444444444444444444444444444444444444444444444444444444444444444444445  (seg0 slice1 172.19.0.4:6434 pid=107133)
+DETAIL:  Command: execute: (echo -n ‘ && seq 1 500 | cut -b1 | tr -d '\n' && echo -n ’) >/dev/stderr && false
+DROP EXTERNAL TABLE IF EXISTS table_test;
+CREATE EXTERNAL WEB TABLE table_test (text text) EXECUTE $$ (echo -n ‘ && seq 1 2006 | cut -b1 | tr -d '\n' && echo -n ’) >/dev/stderr && true $$ ON ALL FORMAT 'CSV';
+SELECT true FROM table_test;
+ bool 
+------
+(0 rows)
+
+SELECT logseverity FROM gp_toolkit.gp_log_system WHERE logmessage LIKE 'read err msg from pipe%' AND logdatabase = current_database() LIMIT 1;
+ logseverity 
+-------------
+ LOG
+(1 row)
+
+DROP EXTERNAL TABLE IF EXISTS table_test;
 -- echo will behave differently on different platforms, force to use bash with -E option
 CREATE EXTERNAL WEB TABLE table_qry (val TEXT)
   EXECUTE E'/usr/bin/env bash -c ''echo -E "$GP_QUERY_STRING"''' ON SEGMENT 0