AI & Agent Dev Bug Sandbox Back to Radar

RecursiveJsonSplitter Incorrectly Sizes Chunks Using Escaped JSON When Ensure_ascii=False

When `ensure_ascii=False` is passed to `split_text`, `_json_size` still measures using `json.dumps` with default `ensure_ascii=True`, causing non-ASCII chunks to be far below `max_chunk_size` and producing more chunks than expected.

mediumConfidence 96%Langchain Affected V1.1.2

Origin Analysis

`_json_size` calls `json.dumps(data)` without forwarding `ensure_ascii`, while `split_text` serializes final chunks with `ensure_ascii=False`.

1. Install langchain-text-splitters 1.1.2.\n2. Run:\n```python\nfrom langchain_text_splitters import RecursiveJsonSplitter\ndata = {str(i): "你好世界" * 10 for i in range(20)}\nsplitter = RecursiveJsonSplitter(max_chunk_size=500)\nchunks = splitter.split_text(json_data=data, ensure_ascii=False)\nprint(len(chunks))\nprint([len(c) for c in chunks])\n```\n3. Observe chunks of length around 98-100 instead of close to 500; with `ensure_ascii=True` chunks are 498-500.

Fixing Code Block

--- a/libs/text-splitters/langchain_text_splitters/json.py
+++ b/libs/text-splitters/langchain_text_splitters/json.py
@@ -51,8 +51,8 @@ class RecursiveJsonSplitter:
-    def _json_size(self, data: dict) -> int:
-        return len(json.dumps(data))
+    def _json_size(self, data: dict, ensure_ascii: bool = True) -> int:
+        return len(json.dumps(data, ensure_ascii=ensure_ascii))
 
-    def _json_split(self, json_data: Dict[str, Any], convert_lists: bool = False) -> List[Dict[str, Any]]:
+    def _json_split(self, json_data: Dict[str, Any], convert_lists: bool = False, ensure_ascii: bool = True) -> List[Dict[str, Any]]:
         if self._json_size(json_data, ensure_ascii=ensure_ascii) <= self.max_chunk_size:
             return [json_data]
 
         if isinstance(json_data, dict):
             chunks = []
             current_chunk: Dict[str, Any] = {}
             for key, value in json_data.items():
                 if isinstance(value, list) and convert_lists:
-                    list_chunks = self._json_split(value, convert_lists=True)
+                    list_chunks = self._json_split(value, convert_lists=True, ensure_ascii=ensure_ascii)
                     for list_chunk in list_chunks:
                         chunk = {key: list_chunk}
-                        if self._json_size(chunk) > self.max_chunk_size:
-                            chunks.extend(self._json_split(chunk, convert_lists=True))
-                        elif self._json_size(current_chunk) + self._json_size(chunk) <= self.max_chunk_size:
+                        if self._json_size(chunk, ensure_ascii=ensure_ascii) > self.max_chunk_size:
+                            chunks.extend(self._json_split(chunk, convert_lists=True, ensure_ascii=ensure_ascii))
+                        elif self._json_size(current_chunk, ensure_ascii=ensure_ascii) + self._json_size(chunk, ensure_ascii=ensure_ascii) <= self.max_chunk_size:
                             current_chunk.update(chunk)
                         else:
                             chunks.append(current_chunk)
                             current_chunk = chunk
                     continue
                 else:
                     chunk = {key: value}
-                if self._json_size(chunk) > self.max_chunk_size:
-                    chunks.extend(self._json_split(chunk, convert_lists=True))
-                elif self._json_size(current_chunk) + self._json_size(chunk) <= self.max_chunk_size:
+                if self._json_size(chunk, ensure_ascii=ensure_ascii) > self.max_chunk_size:
+                    chunks.extend(self._json_split(chunk, convert_lists=True, ensure_ascii=ensure_ascii))
+                elif self._json_size(current_chunk, ensure_ascii=ensure_ascii) + self._json_size(chunk, ensure_ascii=ensure_ascii) <= self.max_chunk_size:
                     current_chunk.update(chunk)
                 else:
                     chunks.append(current_chunk)
                     current_chunk = chunk
             if current_chunk:
                 chunks.append(current_chunk)
             return chunks
         elif isinstance(json_data, list):
             chunks = []
             current_chunk: List[Any] = []
             for item in json_data:
-                item_size = self._json_size({"": item})
+                item_size = self._json_size({"": item}, ensure_ascii=ensure_ascii)
                 if item_size > self.max_chunk_size:
                     if current_chunk:
                         chunks.append(current_chunk)
                         current_chunk = []
-                    chunks.extend(self._json_split(item, convert_lists=True))
+                    chunks.extend(self._json_split(item, convert_lists=True, ensure_ascii=ensure_ascii))
                 elif (
-                    self._json_size({"": current_chunk + [item]})
+                    self._json_size({"": current_chunk + [item]}, ensure_ascii=ensure_ascii)
                     > self.max_chunk_size
                 ):
                     if current_chunk:
                         chunks.append(current_chunk)
                     current_chunk = [item]
                 else:
                     current_chunk.append(item)
             if current_chunk:
                 chunks.append(current_chunk)
             return chunks
         else:
             return [json_data]
 
     def split_json(
-        self, json_data: Dict[str, Any], convert_lists: bool = False
+        self, json_data: Dict[str, Any], convert_lists: bool = False, ensure_ascii: bool = True
     ) -> List[Dict[str, Any]]:
-        return self._json_split(json_data, convert_lists)
+        return self._json_split(json_data, convert_lists, ensure_ascii)
 
     def split_text(
-        self, json_data: Dict[str, Any], convert_lists: bool = False, ensure_ascii: bool = True
+        self, json_data: Dict[str, Any], convert_lists: bool = False, ensure_ascii: bool = True
     ) -> List[str]:
-        chunks = self.split_json(json_data, convert_lists)
+        chunks = self.split_json(json_data, convert_lists, ensure_ascii)
         return [json.dumps(chunk, ensure_ascii=ensure_ascii) for chunk in chunks]

Add `ensure_ascii` parameter to `_json_size`, `_json_split`, and `split_json`, defaulting to `True` to preserve existing behavior. All recursive size checks and split calls now forward `ensure_ascii`, ensuring measurement matches final serialization.

Edge Case Audit

The fix changes chunk sizing for non-ASCII data when `ensure_ascii=False`; chunks will be larger and closer to `max_chunk_size`. If callers unexpectedly relied on the previous smaller chunks, they may see increased per-chunk token counts. Use the default `ensure_ascii=True` to retain escaped sizing. Rollback by reverting this patch or by passing `ensure_ascii=True` explicitly in all calls.