diff --git a/docs/en_US/preferences.rst b/docs/en_US/preferences.rst index 0fcf6413bd5..35afbcb9e5b 100644 --- a/docs/en_US/preferences.rst +++ b/docs/en_US/preferences.rst @@ -589,6 +589,13 @@ Use the fields on the *CSV/TXT Output* panel to control the CSV/TXT output. quoted in the CSV/TXT output; select *Strings*, *All*, or *None*. * Use the *Replace null values with* option to replace null values with specified string in the output file. Default is set to 'NULL'. +* Use the *Output file encoding* drop-down listbox to specify the character + encoding used when saving query results to a file. The default is utf-8; an + encoding that is not listed can also be typed in. +* Use the *Add byte order mark (BOM)?* switch to add a byte order mark at the + start of the saved file when a UTF encoding is used. This helps applications + such as Microsoft Excel detect the encoding correctly. This applies to the + CSV/TXT output only. .. image:: images/preferences_sql_display.png :alt: Preferences sqleditor display options @@ -721,6 +728,9 @@ preferences for copied data. character for copied data. * Use the *Result copy quoting* drop-down listbox to select which type of fields require quoting; select *All*, *None*, or *Strings*. +* When the *Copy with headers?* switch is set to true, the column headers are + included by default when copying data from the results grid. This can still + be toggled per-copy from the results grid copy options menu. * When the *Striped rows?* switch is set to true, the result grid will display rows with alternating background colors. diff --git a/docs/en_US/query_tool_toolbar.rst b/docs/en_US/query_tool_toolbar.rst index f30cc1128ba..e94bbdb3cd4 100644 --- a/docs/en_US/query_tool_toolbar.rst +++ b/docs/en_US/query_tool_toolbar.rst @@ -212,10 +212,12 @@ Data Editing Options | *Save Data Changes* | Click the *Save Data Changes* icon to save data changes (insert, update, or delete) in the Data | F6 | | | Output Panel to the server. | | +----------------------+---------------------------------------------------------------------------------------------------+----------------+ - | *Save results to* | Click the Save results to file icon to save the result set of the current query as a delimited | F8 | - | *file* | text file (CSV, if the field separator is set to a comma). This button will only be enabled when | | - | | a query has been executed and there are results in the data grid. You can specify the CSV/TXT | | - | | settings in the Preference Dialogue under SQL Editor -> CSV/TXT output. | | + | *Save results to* | Click the Save results to file icon to save the result set of the current query. By | F8 | + | *file* | default it is saved as a delimited text file (CSV, if the field separator is set to a | | + | | comma). Use the adjacent drop-down list to instead save the results as JSON or XML. | | + | | This button is only enabled when a query has been executed and there are results in | | + | | the data grid. You can specify the CSV/TXT settings (including the output file encoding | | + | | and byte order mark) in the Preferences dialog under Query Tool -> CSV/TXT Output. | | +----------------------+---------------------------------------------------------------------------------------------------+----------------+ | Graph Visualiser | Use the Graph Visualiser button to generate graphs of the query results. | | +----------------------+---------------------------------------------------------------------------------------------------+----------------+ diff --git a/docs/en_US/release_notes_9_16.rst b/docs/en_US/release_notes_9_16.rst index 4f0740cdff0..4dceb7a91c0 100644 --- a/docs/en_US/release_notes_9_16.rst +++ b/docs/en_US/release_notes_9_16.rst @@ -26,6 +26,10 @@ Bundled PostgreSQL Utilities New features ************ + | `Issue #3205 `_ - Allow saving Query Tool results as JSON and XML in addition to CSV, via a drop-down on the Save results to file button. + | `Issue #4128 `_ - Add a preference to choose the character encoding used when saving Query Tool results to a file. + | `Issue #4129 `_ - Add a "Copy with headers?" preference to control whether column headers are included by default when copying results grid data. + | `Issue #6695 `_ - Add a preference to write a UTF byte order mark (BOM) when saving Query Tool results to a file, for better interoperability with applications such as Microsoft Excel. | `Issue #9626 `_ - Add support for the TOAST tuple target storage parameter in the Materialized View dialog. | `Issue #9646 `_ - Make the init container security context in the Helm chart configurable via containerSecurityContext, consistent with the main container. | `Issue #9699 `_ - Add support for closing a tab with a middle-click on its title. diff --git a/web/pgadmin/tools/sqleditor/__init__.py b/web/pgadmin/tools/sqleditor/__init__.py index c9e26df2f00..f87f6a7ac65 100644 --- a/web/pgadmin/tools/sqleditor/__init__.py +++ b/web/pgadmin/tools/sqleditor/__init__.py @@ -2166,20 +2166,59 @@ def start_query_download_tool(trans_id): } ) + # Output format: csv (default), json or xml. + data_format = (data.get('format') or 'csv').lower() + if data_format not in ('csv', 'json', 'xml'): + data_format = 'csv' + + # Encoding and BOM apply to the CSV/text output only; the structured + # formats are always emitted as UTF-8. + if data_format == 'csv': + output_encoding = blueprint.csv_output_encoding.get() or 'utf-8' + add_bom = blueprint.csv_add_bom.get() + else: + output_encoding = 'utf-8' + add_bom = False + is_utf = output_encoding.lower().replace('-', '').replace( + '_', '').startswith('utf') + + str_gen = gen(conn_obj, + trans_obj, + quote=blueprint.csv_quoting.get(), + quote_char=blueprint.csv_quote_char.get(), + field_separator=blueprint.csv_field_separator.get(), + replace_nulls_with=blueprint.replace_nulls_with.get(), + data_format=data_format) + + def encoded_gen(text_gen): + is_first_chunk = True + for chunk in text_gen: + if is_first_chunk: + is_first_chunk = False + if add_bom and is_utf: + chunk = '\ufeff' + chunk + yield chunk.encode(output_encoding, errors='replace') + + if data_format == 'json': + base_mimetype = 'application/json' + elif data_format == 'xml': + base_mimetype = 'application/xml' + elif blueprint.csv_field_separator.get() == ',': + base_mimetype = 'text/csv' + else: + base_mimetype = 'text/plain' + r = Response( - gen(conn_obj, - trans_obj, - quote=blueprint.csv_quoting.get(), - quote_char=blueprint.csv_quote_char.get(), - field_separator=blueprint.csv_field_separator.get(), - replace_nulls_with=blueprint.replace_nulls_with.get()), - mimetype='text/csv' if - blueprint.csv_field_separator.get() == ',' - else 'text/plain' + encoded_gen(str_gen), + mimetype='{0}; charset={1}'.format(base_mimetype, output_encoding) ) import time - extn = 'csv' if blueprint.csv_field_separator.get() == ',' else 'txt' + if data_format == 'csv': + extn = 'csv' if blueprint.csv_field_separator.get() == ',' \ + else 'txt' + else: + extn = data_format filename = data['filename'] if data.get('filename', '') != "" else \ '{0}.{1}'.format(int(time.time()), extn) diff --git a/web/pgadmin/tools/sqleditor/static/js/components/sections/ResultSet.jsx b/web/pgadmin/tools/sqleditor/static/js/components/sections/ResultSet.jsx index fc6700bee43..ebad870c8a2 100644 --- a/web/pgadmin/tools/sqleditor/static/js/components/sections/ResultSet.jsx +++ b/web/pgadmin/tools/sqleditor/static/js/components/sections/ResultSet.jsx @@ -476,7 +476,8 @@ export class ResultSetUtils { }); } - async saveResultsToFile(fileName, onProgress) { + async saveResultsToFile(fileName, onProgress, dataFormat='csv') { + const mimeTypes = {csv: 'text/csv', json: 'application/json', xml: 'application/xml'}; try { await DownloadUtils.downloadFileStream({ url: url_for('sqleditor.query_tool_download', { @@ -484,8 +485,8 @@ export class ResultSetUtils { }), options: { method: 'POST', - body: JSON.stringify({filename: fileName, query_commited: this.hasQueryCommitted}) - }}, fileName, 'text/csv', onProgress); + body: JSON.stringify({filename: fileName, query_commited: this.hasQueryCommitted, format: dataFormat}) + }}, fileName, mimeTypes[dataFormat] ?? 'text/csv', onProgress); this.eventBus.fireEvent(QUERY_TOOL_EVENTS.TRIGGER_SAVE_RESULTS_END); } catch (error) { this.eventBus.fireEvent(QUERY_TOOL_EVENTS.TRIGGER_SAVE_RESULTS_END); @@ -1052,8 +1053,9 @@ export function ResultSet() { setLoaderText(null); }); - eventBus.registerListener(QUERY_TOOL_EVENTS.TRIGGER_SAVE_RESULTS, async ()=>{ - let extension = queryToolCtx.preferences?.sqleditor?.csv_field_separator === ',' ? '.csv': '.txt'; + eventBus.registerListener(QUERY_TOOL_EVENTS.TRIGGER_SAVE_RESULTS, async (dataFormat='csv')=>{ + const csvExtension = queryToolCtx.preferences?.sqleditor?.csv_field_separator === ',' ? '.csv': '.txt'; + let extension = {csv: csvExtension, json: '.json', xml: '.xml'}[dataFormat] ?? csvExtension; let fileName = 'data-' + new Date().getTime() + extension; if(!queryToolCtx.params.is_query_tool) { fileName = queryToolCtx.params.node_name + extension; @@ -1061,7 +1063,7 @@ export function ResultSet() { setLoaderText(gettext('Downloading results...')); await rsu.current.saveResultsToFile(fileName, (p)=>{ setLoaderText(gettext('Downloading results(%s)...', p)); - }); + }, dataFormat); setLoaderText(''); }); diff --git a/web/pgadmin/tools/sqleditor/static/js/components/sections/ResultSetToolbar.jsx b/web/pgadmin/tools/sqleditor/static/js/components/sections/ResultSetToolbar.jsx index 793ca45dcc4..36daa8b1131 100644 --- a/web/pgadmin/tools/sqleditor/static/js/components/sections/ResultSetToolbar.jsx +++ b/web/pgadmin/tools/sqleditor/static/js/components/sections/ResultSetToolbar.jsx @@ -282,6 +282,7 @@ export function ResultSetToolbar({query, canEdit, totalRowCount, pagination, all /* Menu button refs */ const copyMenuRef = React.useRef(null); const pasetMenuRef = React.useRef(null); + const downloadMenuRef = React.useRef(null); const queryToolPref = queryToolCtx.preferences.sqleditor; @@ -309,8 +310,8 @@ export function ResultSetToolbar({query, canEdit, totalRowCount, pagination, all const addRow = useCallback(()=>{ eventBus.fireEvent(QUERY_TOOL_EVENTS.TRIGGER_ADD_ROWS, [[]], {isNewRow: true}); }, []); - const downloadResult = useCallback(()=>{ - eventBus.fireEvent(QUERY_TOOL_EVENTS.TRIGGER_SAVE_RESULTS); + const downloadResult = useCallback((fmt='csv')=>{ + eventBus.fireEvent(QUERY_TOOL_EVENTS.TRIGGER_SAVE_RESULTS, fmt); }, []); const showGraphVisualiser = useCallback(()=>{ eventBus.fireEvent(QUERY_TOOL_EVENTS.TRIGGER_GRAPH_VISUALISER); @@ -348,6 +349,14 @@ export function ResultSetToolbar({query, canEdit, totalRowCount, pagination, all setDisableButton('save-result', (totalRowCount||0) < 1); }, [totalRowCount]); + useEffect(()=>{ + // Seed the "Copy with headers" toggle default from the user preference. + setCheckedMenuItems((prev)=>({ + ...prev, + copy_with_headers: queryToolPref.copy_column_headers, + })); + }, [queryToolPref.copy_column_headers]); + useEffect(()=>{ eventBus.registerListener(QUERY_TOOL_EVENTS.TRIGGER_COPY_DATA, copyData); return ()=>eventBus.deregisterListener(QUERY_TOOL_EVENTS.TRIGGER_COPY_DATA, copyData); @@ -432,7 +441,10 @@ export function ResultSetToolbar({query, canEdit, totalRowCount, pagination, all } - onClick={downloadResult} shortcut={queryToolPref.download_results} + onClick={()=>downloadResult('csv')} shortcut={queryToolPref.download_results} + disabled={buttonsDisabled['save-result']} /> + } splitButton + name="menu-downloadoptions" ref={downloadMenuRef} onClick={openMenu} disabled={buttonsDisabled['save-result']} /> @@ -490,6 +502,16 @@ export function ResultSetToolbar({query, canEdit, totalRowCount, pagination, all > {gettext('Paste with SERIAL/IDENTITY values?')} + + downloadResult('csv')}>{gettext('Save as CSV/Text')} + downloadResult('json')}>{gettext('Save as JSON')} + downloadResult('xml')}>{gettext('Save as XML')} + ); } diff --git a/web/pgadmin/tools/sqleditor/tests/test_download_csv_query_tool.py b/web/pgadmin/tools/sqleditor/tests/test_download_csv_query_tool.py index f4157dbdc7f..17a1bd28f44 100644 --- a/web/pgadmin/tools/sqleditor/tests/test_download_csv_query_tool.py +++ b/web/pgadmin/tools/sqleditor/tests/test_download_csv_query_tool.py @@ -240,3 +240,133 @@ def tearDown(self): self.server['sslmode'] ) test_utils.drop_database(main_conn, self._db_name) + + +class TestDownloadResultFormats(BaseTestGenerator): + """ + Validates downloading query results as JSON and XML, the UTF BOM option + and the output file encoding option. + """ + SQL = 'SELECT 1 as "A", 2 as "B", \'x\' as "C"' + INIT_URL = '/sqleditor/initialize/sqleditor/{0}/{1}/{2}/{3}' + DOWNLOAD_URL = '/sqleditor/query_tool/download/{0}' + + scenarios = [ + ( + 'Download results as JSON', + dict(data_format='json', add_bom=False, encoding='utf-8', + expected_content_type='application/json', + expected_extension='.json') + ), + ( + 'Download results as XML', + dict(data_format='xml', add_bom=False, encoding='utf-8', + expected_content_type='application/xml', + expected_extension='.xml') + ), + ( + 'Download CSV with a UTF BOM', + dict(data_format='csv', add_bom=True, encoding='utf-8', + expected_content_type='text/csv', + expected_extension='.csv') + ), + ( + 'Download CSV with a non-UTF output encoding', + dict(data_format='csv', add_bom=True, encoding='latin-1', + expected_content_type='text/csv', + expected_extension='.csv') + ), + ] + + def setUp(self): + self._db_name = 'download_results_fmt_' + str( + secrets.choice(range(10000, 65535))) + self._sid = self.server_information['server_id'] + server_utils.connect_server(self, self._sid) + self._did = test_utils.create_database(self.server, self._db_name) + + def initiate_sql_query_tool(self, trans_id, sql_query): + url = '/sqleditor/query_tool/start/{0}'.format(trans_id) + response = self.tester.post(url, data=json.dumps({"sql": sql_query}), + content_type='html/json') + self.assertEqual(response.status_code, 200) + return async_poll(tester=self.tester, + poll_url='/sqleditor/poll/{0}'.format(trans_id)) + + def runTest(self): + db_con = database_utils.connect_database(self, + test_utils.SERVER_GROUP, + self._sid, + self._did) + if not db_con["info"] == "Database connected.": + raise Exception("Could not connect to the database.") + + self.trans_id = str(secrets.choice(range(1, 9999999))) + url = self.INIT_URL.format( + self.trans_id, test_utils.SERVER_GROUP, self._sid, self._did) + response = self.tester.post(url, data=json.dumps({ + "dbname": self._db_name + })) + self.assertEqual(response.status_code, 200) + + self.initiate_sql_query_tool(self.trans_id, self.SQL) + + url = self.DOWNLOAD_URL.format(self.trans_id) + self.app.logger.disabled = True + filename = 'test{0}'.format(self.expected_extension) + with patch('pgadmin.tools.sqleditor.blueprint.' + 'csv_add_bom.get', return_value=self.add_bom), \ + patch('pgadmin.tools.sqleditor.blueprint.' + 'csv_output_encoding.get', return_value=self.encoding): + response = self.tester.post(url, data={ + "query": self.SQL, + "filename": filename, + "format": self.data_format, + "query_commited": True, + }) + self.app.logger.disabled = False + + headers = dict(response.headers) + self.assertEqual(response.status_code, 200) + self.assertIn(self.expected_content_type, headers['Content-Type']) + self.assertIn('charset={0}'.format(self.encoding), + headers['Content-Type']) + self.assertIn(filename, headers['Content-Disposition']) + + raw = response.data + if self.add_bom and self.encoding.lower().startswith('utf'): + self.assertTrue(raw.startswith(b'\xef\xbb\xbf')) + else: + self.assertFalse(raw.startswith(b'\xef\xbb\xbf')) + + body = raw.decode(self.encoding) + + if self.data_format == 'json': + parsed = json.loads(body) + self.assertIsInstance(parsed, list) + self.assertEqual(parsed[0]['A'], 1) + self.assertEqual(parsed[0]['B'], 2) + self.assertEqual(parsed[0]['C'], 'x') + elif self.data_format == 'xml': + self.assertIn('', body) + self.assertIn('1', body) + self.assertIn('x', body) + self.assertIn('', body) + else: + self.assertIn('"A","B","C"', body) + + url = '/sqleditor/close/{0}'.format(self.trans_id) + response = self.tester.delete(url) + self.assertEqual(response.status_code, 200) + database_utils.disconnect_database(self, self._sid, self._did) + + def tearDown(self): + main_conn = test_utils.get_db_connection( + self.server['db'], + self.server['username'], + self.server['db_password'], + self.server['host'], + self.server['port'], + self.server['sslmode'] + ) + test_utils.drop_database(main_conn, self._db_name) diff --git a/web/pgadmin/tools/sqleditor/utils/query_tool_preferences.py b/web/pgadmin/tools/sqleditor/utils/query_tool_preferences.py index a18edb13843..79cf3d6c677 100644 --- a/web/pgadmin/tools/sqleditor/utils/query_tool_preferences.py +++ b/web/pgadmin/tools/sqleditor/utils/query_tool_preferences.py @@ -250,6 +250,34 @@ def register_query_tool_preferences(self): allow_blanks=True ) + self.csv_output_encoding = self.preference.register( + 'CSV_output', 'csv_output_encoding', + gettext("Output file encoding"), 'options', 'utf-8', + category_label=PREF_LABEL_CSV_TXT, + options=[{'label': 'utf-8', 'value': 'utf-8'}, + {'label': 'utf-16', 'value': 'utf-16'}, + {'label': 'latin-1', 'value': 'latin-1'}, + {'label': 'windows-1252', 'value': 'windows-1252'}], + control_props={ + 'allowClear': False, + 'tags': False, + 'creatable': True + }, + help_str=gettext('The character encoding used when saving query ' + 'results to a file. Defaults to utf-8. A different ' + 'encoding can be typed in if it is not listed.') + ) + + self.csv_add_bom = self.preference.register( + 'CSV_output', 'csv_add_bom', + gettext("Add byte order mark (BOM)?"), 'boolean', + False, category_label=PREF_LABEL_CSV_TXT, + help_str=gettext('If set to True, a byte order mark (BOM) is added at ' + 'the start of the saved file when a UTF encoding is ' + 'used. This helps applications such as Microsoft ' + 'Excel detect the encoding correctly.') + ) + self.results_grid_quoting = self.preference.register( 'Results_grid', 'results_grid_quoting', gettext("Result copy quoting"), 'options', 'strings', @@ -290,6 +318,16 @@ def register_query_tool_preferences(self): } ) + self.copy_column_headers = self.preference.register( + 'Results_grid', 'copy_column_headers', + gettext("Copy with headers?"), 'boolean', + False, category_label=PREF_LABEL_RESULTS_GRID, + help_str=gettext('If set to True, the column headers are included by ' + 'default when copying data from the results grid. ' + 'This can still be toggled per-copy from the results ' + 'grid copy menu.') + ) + self.column_data_auto_resize = self.preference.register( 'Results_grid', 'column_data_auto_resize', gettext("Columns sized by"), 'radioModern', 'by_data', diff --git a/web/pgadmin/utils/driver/psycopg3/connection.py b/web/pgadmin/utils/driver/psycopg3/connection.py index 47bf8503b91..2f829412d1e 100644 --- a/web/pgadmin/utils/driver/psycopg3/connection.py +++ b/web/pgadmin/utils/driver/psycopg3/connection.py @@ -17,7 +17,9 @@ import secrets import datetime import asyncio +import json from collections import deque +from xml.sax.saxutils import escape as xml_escape, quoteattr as xml_quoteattr import psycopg from flask import g, current_app from flask_babel import gettext @@ -53,6 +55,62 @@ _ = gettext + +def _json_default(value): + """Fallback serialiser for values that json cannot encode natively + (dates, Decimals, intervals, etc.).""" + return str(value) + + +def _generate_json(cur, records, results, header, replace_nulls_with, + handle_null_values): + """Stream the result set as a JSON array of row objects. + + The first batch of rows (``results``) has already been fetched by the + caller; subsequent batches are pulled with ``fetchmany(records)``. + """ + yield '[' + is_first_row = True + while results: + if replace_nulls_with is not None: + results = handle_null_values(results, replace_nulls_with) + for row in results: + row_json = json.dumps(dict(row), default=_json_default) + yield row_json if is_first_row else ',' + row_json + is_first_row = False + results = cur.fetchmany(records) + yield ']' + + +def _generate_xml(cur, records, results, header, replace_nulls_with, + handle_null_values): + """Stream the result set as XML. + + Column names are emitted as escaped ``name`` attributes (rather than + element names) so that column names which are not valid XML element + names are handled safely. + """ + yield '\n' + while results: + if replace_nulls_with is not None: + results = handle_null_values(results, replace_nulls_with) + for row in results: + row_io = [''] + for column in header: + value = row.get(column) + if value is None: + row_io.append( + ''.format( + xml_quoteattr(column))) + else: + row_io.append('{1}'.format( + xml_quoteattr(column), xml_escape(str(value)))) + row_io.append('') + yield ''.join(row_io) + results = cur.fetchmany(records) + yield '' + + # Register global type caster which will be applicable to all connections. register_global_typecasters() configure_driver_encodings(encodings) @@ -912,7 +970,8 @@ def handle_null_values(results, replace_nulls_with): return results def gen(conn_obj, trans_obj, quote='strings', quote_char="'", - field_separator=',', replace_nulls_with=None): + field_separator=',', replace_nulls_with=None, + data_format='csv'): try: cur.scroll(0, mode='absolute') @@ -935,37 +994,24 @@ def gen(conn_obj, trans_obj, quote='strings', quote_char="'", if c.to_dict()['type_code'] in ALL_JSON_TYPES: json_columns.append(column_name) - res_io = StringIO() - - if quote == 'strings': - quote = csv.QUOTE_NONNUMERIC - elif quote == 'all': - quote = csv.QUOTE_ALL + if data_format == 'json': + yield from _generate_json(cur, records, results, header, + replace_nulls_with, + handle_null_values) + elif data_format == 'xml': + yield from _generate_xml(cur, records, results, header, + replace_nulls_with, + handle_null_values) else: - quote = csv.QUOTE_NONE - - csv_writer = csv.DictWriter( - res_io, fieldnames=header, delimiter=field_separator, - quoting=quote, - quotechar=quote_char, - replace_nulls_with=replace_nulls_with - ) - - csv_writer.writeheader() - # Replace the null values with given string if configured. - if replace_nulls_with is not None: - results = handle_null_values(results, replace_nulls_with) - csv_writer.writerows(results) - - yield res_io.getvalue() - - while True: - results = cur.fetchmany(records) - - if not results: - break res_io = StringIO() + if quote == 'strings': + quote = csv.QUOTE_NONNUMERIC + elif quote == 'all': + quote = csv.QUOTE_ALL + else: + quote = csv.QUOTE_NONE + csv_writer = csv.DictWriter( res_io, fieldnames=header, delimiter=field_separator, quoting=quote, @@ -973,12 +1019,35 @@ def gen(conn_obj, trans_obj, quote='strings', quote_char="'", replace_nulls_with=replace_nulls_with ) + csv_writer.writeheader() # Replace the null values with given string if configured. if replace_nulls_with is not None: results = handle_null_values(results, replace_nulls_with) csv_writer.writerows(results) + yield res_io.getvalue() + while True: + results = cur.fetchmany(records) + + if not results: + break + res_io = StringIO() + + csv_writer = csv.DictWriter( + res_io, fieldnames=header, delimiter=field_separator, + quoting=quote, + quotechar=quote_char, + replace_nulls_with=replace_nulls_with + ) + + # Replace the null values with given string if configured. + if replace_nulls_with is not None: + results = handle_null_values(results, + replace_nulls_with) + csv_writer.writerows(results) + yield res_io.getvalue() + try: # try to reset the cursor scroll back to where it was, # bypass error, if cannot scroll back