WIP SQL console: export data

src/main/java/tech/ailef/dbadmin/external/controller/DataExportController.java

@@ -41,13 +41,17 @@ import tech.ailef.dbadmin.external.dbmapping.DbField;
 import tech.ailef.dbadmin.external.dbmapping.DbFieldValue;
 import tech.ailef.dbadmin.external.dbmapping.DbObject;
 import tech.ailef.dbadmin.external.dbmapping.DbObjectSchema;
+import tech.ailef.dbadmin.external.dbmapping.query.DbQueryResult;
+import tech.ailef.dbadmin.external.dbmapping.query.DbQueryResultRow;
 import tech.ailef.dbadmin.external.dto.DataExportFormat;
 import tech.ailef.dbadmin.external.dto.QueryFilter;
 import tech.ailef.dbadmin.external.exceptions.DbAdminException;
 import tech.ailef.dbadmin.external.misc.Utils;
+import tech.ailef.dbadmin.internal.model.ConsoleQuery;
+import tech.ailef.dbadmin.internal.repository.ConsoleQueryRepository;
 
 @Controller
-@RequestMapping(value = { "/${dbadmin.baseUrl}/export", "/${dbadmin.baseUrl}/export/" })
+@RequestMapping(value = { "/${dbadmin.baseUrl}/", "/${dbadmin.baseUrl}" })
 public class DataExportController {
 	private static final Logger logger = LoggerFactory.getLogger(DataExportFormat.class);
 	
@@ -57,10 +61,50 @@ public class DataExportController {
 	@Autowired
 	private DbAdminRepository repository;
 	
+	@Autowired
+	private ConsoleQueryRepository queryRepository;
+	
 	@Autowired
 	private ObjectMapper mapper;
 
-	@GetMapping("/{className}")
+	@GetMapping("/console/export/{queryId}")
+	public ResponseEntity<byte[]> export(@PathVariable String queryId, @RequestParam String format, 
+			@RequestParam MultiValueMap<String, String> otherParams) {
+		ConsoleQuery query = queryRepository.findById(queryId).orElseThrow(() -> new DbAdminException("Query not found: " + queryId));
+		
+		DataExportFormat exportFormat = null;
+		try {
+			exportFormat = DataExportFormat.valueOf(format.toUpperCase());
+		} catch (IllegalArgumentException e) {
+			throw new DbAdminException("Unsupported export format: " + format);
+		}
+		
+		List<String> fieldsToInclude = otherParams.getOrDefault("fields[]", new ArrayList<>());
+		DbQueryResult results = repository.executeQuery(query.getSql());
+		
+		switch (exportFormat) {
+		case CSV:
+			return ResponseEntity.ok()
+					.header(HttpHeaders.CONTENT_DISPOSITION,
+							"attachment; filename=\"export_" + query.getTitle().replaceAll("[^a-zA-Z0-9.-]", "_") + ".csv\"")
+					.body(toCsvQuery(results, fieldsToInclude).getBytes());
+		case XLSX:
+			String sheetName = query.getTitle();
+			return ResponseEntity.ok()
+					.header(HttpHeaders.CONTENT_DISPOSITION,
+					"attachment; filename=\"export_" + query.getTitle().replaceAll("[^a-zA-Z0-9.-]", "_") + ".xlsx\"")
+					.body(toXlsxQuery(sheetName, results, fieldsToInclude));
+		case JSONL:
+			return ResponseEntity.ok()
+					.header(HttpHeaders.CONTENT_DISPOSITION,
+							"attachment; filename=\"export_" + query.getTitle().replaceAll("[^a-zA-Z0-9.-]", "_") + ".jsonl\"")
+					.body(toJsonlQuery(results, fieldsToInclude).getBytes());
+		default:
+			throw new DbAdminException("Invalid DataExportFormat");
+		}
+	}
+	
+	@GetMapping("/export/{className}")
 	@ResponseBody
 	public ResponseEntity<byte[]> export(@PathVariable String className, @RequestParam(required = false) String query,
 			@RequestParam String format, @RequestParam(required=false) Boolean raw, 
@@ -149,6 +193,49 @@ public class DataExportController {
 		}
 		
 		
+		return fos.toByteArray();
+	}
+	
+	private byte[] toXlsxQuery(String sheetName, DbQueryResult result, List<String> fields) {
+		Workbook workbook = new XSSFWorkbook();
+
+		Sheet sheet = workbook.createSheet(sheetName);
+
+		CellStyle headerStyle = workbook.createCellStyle();
+		Font headerFont = workbook.createFont();
+		headerFont.setBold(true);
+		headerStyle.setFont(headerFont);
+		
+		int rowIndex = 0;
+		Row headerRow = sheet.createRow(rowIndex++);
+		for (int i = 0; i < fields.size(); i++) {
+			Cell headerCell = headerRow.createCell(i);
+			headerCell.setCellValue(fields.get(i));
+			headerCell.setCellStyle(headerStyle);
+		}
+		
+		for (DbQueryResultRow item : result.getRows()) {
+			Row row = sheet.createRow(rowIndex++);
+			int cellIndex = 0;
+			
+			List<String> record = getRecord(item, fields);
+			
+			for (String value : record) {
+				Cell cell = row.createCell(cellIndex++);
+				cell.setCellValue(value);
+			}
+		}
+
+		ByteArrayOutputStream fos = new ByteArrayOutputStream();
+		try {
+			workbook.write(fos);
+			fos.close();
+			workbook.close();
+		} catch (IOException e) {
+			throw new DbAdminException("Error during serialization for XLSX workbook", e);
+		}
+		
+		
 		return fos.toByteArray();
 	}
 
@@ -181,6 +268,28 @@ public class DataExportController {
 		return sb.toString();
 	}
 	
+	private String toJsonlQuery(DbQueryResult result, List<String> fields) {
+		if (result.isEmpty())
+			return "";
+
+		StringBuilder sb = new StringBuilder();
+		
+		for (DbQueryResultRow item : result.getRows()) {
+			Map<String, Object> map = item.toMap(fields);
+			try {
+				String json = mapper.writeValueAsString(map);
+				sb.append(json);
+			} catch (JsonProcessingException e) {
+				throw new DbAdminException(e);
+			}
+			
+			sb.append("\n");
+		}
+
+		return sb.toString();
+	
+	}
+	
 	private String toCsv(List<DbObject> items, List<String> fields, boolean raw) {
 		if (items.isEmpty())
 			return "";
@@ -203,6 +312,40 @@ public class DataExportController {
 		}
 	}
 	
+	private String toCsvQuery(DbQueryResult result, List<String> fields) {
+		if (result.isEmpty())
+			return "";
+
+		StringWriter sw = new StringWriter();
+
+		CSVFormat csvFormat = 
+			CSVFormat.DEFAULT.builder()
+					 .setHeader(fields.toArray(String[]::new))
+					 .build();
+
+		try (final CSVPrinter printer = new CSVPrinter(sw, csvFormat)) {
+			for (DbQueryResultRow item : result.getRows()) {
+				printer.printRecord(getRecord(item, fields));
+			}
+
+			return sw.toString();
+		} catch (IOException e) {
+			throw new DbAdminException("Error during creation of CSV file", e);
+		}
+	
+	}
+	
+	private List<String> getRecord(DbQueryResultRow row, List<String> fields) {
+		List<String> record = new ArrayList<>();
+		
+		for (String field : fields) {
+			Object value = row.getFieldByName(field);
+			record.add(value ==  null ? null : value.toString());
+		}
+		
+		return record;
+	}
+	
 	/**
 	 * Builds and returns a record (i.e a row) for a spreadsheet file (CSV or XLSX) as a list of Strings.
 	 * Each column contains the value of a database column, potentially with some processing applied if

src/main/java/tech/ailef/dbadmin/external/controller/DefaultDbAdminController.java

@@ -19,7 +19,6 @@
 
 package tech.ailef.dbadmin.external.controller;
 
-import java.sql.ResultSetMetaData;
 import java.text.DecimalFormat;
 import java.time.LocalDateTime;
 import java.util.ArrayList;
@@ -32,7 +31,6 @@ import java.util.stream.Collectors;
 
 import org.hibernate.id.IdentifierGenerationException;
 import org.springframework.beans.factory.annotation.Autowired;
-import org.springframework.dao.DataAccessException;
 import org.springframework.dao.DataIntegrityViolationException;
 import org.springframework.http.HttpStatus;
 import org.springframework.jdbc.UncategorizedSQLException;
@@ -59,9 +57,7 @@ import tech.ailef.dbadmin.external.DbAdminProperties;
 import tech.ailef.dbadmin.external.dbmapping.DbAdminRepository;
 import tech.ailef.dbadmin.external.dbmapping.DbObject;
 import tech.ailef.dbadmin.external.dbmapping.DbObjectSchema;
-import tech.ailef.dbadmin.external.dbmapping.query.DbQueryOutputField;
 import tech.ailef.dbadmin.external.dbmapping.query.DbQueryResult;
-import tech.ailef.dbadmin.external.dbmapping.query.DbQueryResultRow;
 import tech.ailef.dbadmin.external.dto.CompareOperator;
 import tech.ailef.dbadmin.external.dto.FacetedSearchRequest;
 import tech.ailef.dbadmin.external.dto.LogsSearchRequest;
@@ -101,9 +97,6 @@ public class DefaultDbAdminController {
 	@Autowired
 	private ConsoleQueryRepository consoleQueryRepository;
 	
-	@Autowired
-	private JdbcTemplate jdbcTemplate; 
-
 	@Autowired
 	private UserSettingsRepository userSettingsRepo;
 	
@@ -589,8 +582,6 @@ public class DefaultDbAdminController {
 		return "redirect:/" + properties.getBaseUrl() + "/console";
 	}
 	
-	
-	
 	@GetMapping("/console/run/{queryId}")
 	public String consoleRun(Model model, @RequestParam(required = false) String query,
 			@RequestParam(required = false) String queryTitle,
@@ -626,36 +617,7 @@ public class DefaultDbAdminController {
 		List<ConsoleQuery> tabs = consoleQueryRepository.findAll();
 		model.addAttribute("tabs", tabs);
 		
-		List<DbQueryResultRow> results = new ArrayList<>();
+		DbQueryResult results = repository.executeQuery(activeQuery.getSql());
-		if (activeQuery.getSql() != null && !activeQuery.getSql().isBlank()) {
-			try {
-				results = jdbcTemplate.query(activeQuery.getSql(), (rs, rowNum) -> {
-					Map<DbQueryOutputField, Object> result = new HashMap<>();
-					
-					ResultSetMetaData metaData = rs.getMetaData();
-					int cols = metaData.getColumnCount();
-					
-					for (int i = 0; i < cols; i++) {
-						Object o = rs.getObject(i + 1);
-						String columnName = metaData.getColumnName(i + 1);
-						String tableName = metaData.getTableName(i + 1);
-						DbQueryOutputField field = new DbQueryOutputField(columnName, tableName, dbAdmin);
-						
-						result.put(field, o);
-					}
-					
-					DbQueryResultRow row = new DbQueryResultRow(result, query);
-					
-					result.keySet().forEach(f -> {
-						f.setResult(row);
-					});
-					
-					return row;
-				});
-			} catch (DataAccessException e) {
-				model.addAttribute("error", e.getMessage());
-			}
-		}
 		
 		if (!results.isEmpty()) {
 			int maxPage = (int)(Math.ceil ((double)results.size() / pageSize));
@@ -665,9 +627,9 @@ public class DefaultDbAdminController {
 			
 			endOffset = Math.min(results.size(), endOffset);
 			
-			results = results.subList(startOffset, endOffset);
+			results.crop(startOffset, endOffset);
 			model.addAttribute("pagination", pagination);
-			model.addAttribute("results", new DbQueryResult(results));
+			model.addAttribute("results", results);
 		}
 		
 		model.addAttribute("title", "SQL Console | " + activeQuery.getTitle());
@@ -675,6 +637,7 @@ public class DefaultDbAdminController {
 		model.addAttribute("elapsedTime", new DecimalFormat("0.0#").format(elapsedTime));
 		return "console";
 	}
+
 	
 	@GetMapping("/settings/appearance")
 	public String settingsAppearance(Model model) {

src/main/java/tech/ailef/dbadmin/external/dbmapping/DbAdminRepository.java

@@ -19,7 +19,9 @@
 
 package tech.ailef.dbadmin.external.dbmapping;
 
+import java.sql.ResultSetMetaData;
 import java.util.ArrayList;
+import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
 import java.util.Objects;
@@ -27,10 +29,12 @@ import java.util.Optional;
 import java.util.Set;
 import java.util.stream.Collectors;
 
+import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.data.domain.Page;
 import org.springframework.data.domain.PageRequest;
 import org.springframework.data.domain.Sort;
 import org.springframework.data.jpa.repository.support.SimpleJpaRepository;
+import org.springframework.jdbc.core.JdbcTemplate;
 import org.springframework.stereotype.Component;
 import org.springframework.transaction.annotation.Transactional;
 import org.springframework.web.multipart.MultipartFile;
@@ -39,7 +43,11 @@ import jakarta.validation.ConstraintViolation;
 import jakarta.validation.ConstraintViolationException;
 import jakarta.validation.Validation;
 import jakarta.validation.Validator;
+import tech.ailef.dbadmin.external.DbAdmin;
 import tech.ailef.dbadmin.external.annotations.ReadOnly;
+import tech.ailef.dbadmin.external.dbmapping.query.DbQueryOutputField;
+import tech.ailef.dbadmin.external.dbmapping.query.DbQueryResult;
+import tech.ailef.dbadmin.external.dbmapping.query.DbQueryResultRow;
 import tech.ailef.dbadmin.external.dto.FacetedSearchRequest;
 import tech.ailef.dbadmin.external.dto.PaginatedResult;
 import tech.ailef.dbadmin.external.dto.PaginationInfo;
@@ -52,6 +60,12 @@ import tech.ailef.dbadmin.external.exceptions.InvalidPageException;
  */
 @Component
 public class DbAdminRepository {
+	@Autowired
+	private JdbcTemplate jdbcTemplate;
+	
+	@Autowired
+	private DbAdmin dbAdmin;
+	
 	public DbAdminRepository() {
 	}
 
@@ -272,6 +286,39 @@ public class DbAdminRepository {
 					.toList();
 	}
 	
+	/**
+	 * Execute custom SQL query using jdbcTemplate
+	 */
+	public DbQueryResult executeQuery(String sql) {
+		List<DbQueryResultRow> results = new ArrayList<>();
+		if (sql != null && !sql.isBlank()) {
+			results = jdbcTemplate.query(sql, (rs, rowNum) -> {
+				Map<DbQueryOutputField, Object> result = new HashMap<>();
+				
+				ResultSetMetaData metaData = rs.getMetaData();
+				int cols = metaData.getColumnCount();
+				
+				for (int i = 0; i < cols; i++) {
+					Object o = rs.getObject(i + 1);
+					String columnName = metaData.getColumnName(i + 1);
+					String tableName = metaData.getTableName(i + 1);
+					DbQueryOutputField field = new DbQueryOutputField(columnName, tableName, dbAdmin);
+					
+					result.put(field, o);
+				}
+				
+				DbQueryResultRow row = new DbQueryResultRow(result, sql);
+				
+				result.keySet().forEach(f -> {
+					f.setResult(row);
+				});
+				
+				return row;
+			});
+		}
+		return new DbQueryResult(results);
+	}
+	
 	/**
 	 * Delete a specific object
 	 * @param schema

src/main/java/tech/ailef/dbadmin/external/dbmapping/query/DbQueryOutputField.java

@@ -64,6 +64,11 @@ public class DbQueryOutputField {
 	public boolean isForeignKey() {
 		return dbField != null && dbField.isForeignKey();
 	}
+
+	public boolean isExportable() {
+		if (dbField == null) return true;
+		return dbField.isExportable();
+	}
 	
 	public Class<?> getConnectedType() {
 		if (dbField == null) return null;

src/main/java/tech/ailef/dbadmin/external/dbmapping/query/DbQueryResult.java

@@ -29,4 +29,8 @@ public class DbQueryResult {
 	public int size() {
 		return rows.size();
 	}
+	
+	public void crop(int startOffset, int endOffset) {
+		rows = rows.subList(startOffset, endOffset);
+	}
 }

src/main/java/tech/ailef/dbadmin/external/dbmapping/query/DbQueryResultRow.java

@@ -1,8 +1,11 @@
 package tech.ailef.dbadmin.external.dbmapping.query;
 
+import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
 
+import tech.ailef.dbadmin.external.exceptions.DbAdminException;
+
 public class DbQueryResultRow {
 	private Map<DbQueryOutputField, Object> values;
 	
@@ -32,8 +35,24 @@ public class DbQueryResultRow {
 	public Object get(DbQueryOutputField field) {
 		return values.get(field);
 	}
+
+	public Object getFieldByName(String field) {
+		DbQueryOutputField key = 
+			values.keySet().stream().filter(f -> f.getName().equals(field)).findFirst().orElse(null);
+		if (key == null) {
+			throw new DbAdminException("Field " + field + " not found");
+		}
+		return get(key);
+	}
 	
-	
+	public Map<String, Object> toMap(List<String> fields) {
+		Map<String, Object> result = new HashMap<>();
+		for (String field : fields) {
+			result.put(field, getFieldByName(field));
+		}
+		return result;
+		
+	}
 	
 	
 }

src/main/resources/templates/console.html

@@ -3,6 +3,60 @@
 	<head th:replace="~{fragments/resources::head}">
 	</head>
 	<body>
+		<!-- Modal -->
+		<div class="modal modal-lg fade" id="csvQueryExportModal" tabindex="-1" aria-labelledby="csvQueryExportModalLabel" aria-hidden="true"
+			th:if="${results != null}">
+			<form th:action="|/${dbadmin_baseUrl}/console/export/${activeQuery.getId()}|" method="GET">
+				<div class="modal-dialog">
+					<div class="modal-content">
+						<div class="modal-header">
+							<h1 class="modal-title fs-5" id="csvQueryExportModalLabel">Export settings</h1>
+							<button type="button" class="btn-close" data-bs-dismiss="modal" aria-label="Close"></button>
+						</div>
+						<div class="modal-body">
+							<div class="container-fluid">
+								<p class="text-muted">The export file will contain all the pages in the results. If the table is big,
+								this might take some time.</p>
+								
+								<h5 class="fw-bold">Include columns</h5>
+								<div th:each="field : ${results.getSortedFields()}" th:if="${field.isExportable()}">
+									<div class="form-check">
+										<input class="form-check-input" type="checkbox" 
+											th:value="${field.getName()}" th:id="|__check_${field.getName()}|"
+											th:name="fields[]" 
+											checked>
+										<label class="form-check-label" th:for="|__check_${field.getName()}|">
+									    	[[ ${field.getName()} ]]
+										</label>
+									</div>
+								</div>
+								<h5 class="fw-bold mt-3">Export format</h3>
+								<select name="format" class="form-select">
+									<option th:each="format : ${T(tech.ailef.dbadmin.external.dto.DataExportFormat).values()}"
+										th:value="${format}" th:text="${format}">
+									</option>
+								</select>
+								<div class="form-check mt-3">
+									<input class="form-check-input" type="checkbox" checked="checked"
+										disabled 
+										id="__check_raw"
+										th:name="raw">
+									<label class="form-check-label" for="__check_raw">
+								    	Export raw values
+									</label>
+								</div>
+							</div>
+						</div>
+						<div class="modal-footer">
+							<button type="button" class="btn btn-secondary" data-bs-dismiss="modal">Cancel</button>
+							<button type="submit" class="btn btn-primary">Export [[ ${pagination.getMaxElement()} ]] rows</button>
+						</div>
+					</div>
+				</div>
+			</form>
+		</div>
+		<!-- End modal -->
+		
     	<div class="bg-light main-wrapper">
     		<nav th:replace="~{fragments/resources :: navbar}"></nav>
 	        <div class="d-flex">
@@ -92,6 +146,10 @@
 														<i>Showing [[ ${results.size()} ]] of [[ ${pagination.getMaxElement()} ]] 
 															results in [[ ${elapsedTime} ]] seconds</i>
 													</p>
+													<button th:if="${results != null}" title="Open export data window" type="button" 
+														class="btn pb-0 pt-0"  data-bs-toggle="modal" data-bs-target="#csvQueryExportModal">
+														<i class="bi bi-file-earmark-spreadsheet export-icon" style="font-size: 1.6rem;"></i>
+													</button>
 												</div>
 											</div>
 											
@@ -100,13 +158,17 @@
 													<i>Showing [[ ${results.size()} ]] of [[ ${pagination.getMaxElement()} ]] 
 														results in [[ ${elapsedTime} ]] seconds</i>
 												</p>
+												<button th:if="${results != null}" title="Open export data window" type="button" 
+													class="btn pb-0 pt-0"  data-bs-toggle="modal" data-bs-target="#csvQueryExportModal">
+													<i class="bi bi-file-earmark-spreadsheet export-icon" style="font-size: 1.6rem;"></i>
+												</button>
 											</div>
 										</div>
 									</nav>
 								
-
 									
-	        						<div th:replace="~{fragments/generic_table :: table(results=${results})}"></div>
+									
+									<div th:replace="~{fragments/generic_table :: table(results=${results})}"></div>
 	        						
 	        						
 	        						
@@ -163,9 +225,9 @@
 												</p>
 											</div>
 										</div>
+										
 									</nav>
 								
-	        						
 	        					</div>
 	        					<div th:if="${error != null}">
 									<div class="separator mt-3 mb-3"></div>

src/main/resources/templates/fragments/generic_data_row.html

@@ -24,8 +24,6 @@
 						<a th:href="|/${dbadmin_baseUrl}/model/${field.getConnectedType().getName()}/show/${object.get(field)}|">
 							<span th:text="${object.get(field)}"></span>
 						</a>
-						<p class="p-0 m-0"
-							th:text="${object.traverse(field).getDisplayName()}"></p>
 					</th:block>
 					<th:block th:if="${!field.isForeignKey()}">
 						<span th:text="${object.get(field)}"></span>