From 66720409bdc22b71b5537124cdd694eb3ca2e478 Mon Sep 17 00:00:00 2001
From: yangdx <yangdx@znipower.com>
Date: Mon, 27 Jan 2025 15:32:27 +0800
Subject: [PATCH] Enhance robustness of CVS processing ,Fix potential CSV
 parsing issues

- Add CSV quoting for all fields
- Remove null characters from CSV input
- Improve CSV data integrity
- Ensure consistent CSV formatting
---
 lightrag/utils.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/lightrag/utils.py b/lightrag/utils.py
index 3454ea7c..86867c9c 100644
--- a/lightrag/utils.py
+++ b/lightrag/utils.py
@@ -237,13 +237,13 @@ def truncate_list_by_token_size(list_data: list, key: callable, max_token_size:
 
 def list_of_list_to_csv(data: List[List[str]]) -> str:
     output = io.StringIO()
-    writer = csv.writer(output)
+    writer = csv.writer(output, quoting=csv.QUOTE_ALL)
     writer.writerows(data)
     return output.getvalue()
 
 
 def csv_string_to_list(csv_string: str) -> List[List[str]]:
-    output = io.StringIO(csv_string)
+    output = io.StringIO(csv_string.replace('\x00', ''))
     reader = csv.reader(output)
     return [row for row in reader]