Workaround for incremental in read_json. (apache#13489)

chenghxy · Dec 12, 2020 · 302de87 · 302de87
1 parent ee911d2
commit 302de87
Showing 1 changed file with 5 additions and 0 deletions.
diff --git a/sdks/python/apache_beam/dataframe/io.py b/sdks/python/apache_beam/dataframe/io.py
@@ -59,6 +59,11 @@ def read_fwf(path, *args, **kwargs):
 
 
 def read_json(path, *args, **kwargs):
+  if 'nrows' in kwargs:
+    raise NotImplementedError('nrows not yet supported')
+  elif kwargs.get('lines', False):
+    # Work around https://github.com/pandas-dev/pandas/issues/34548.
+    kwargs = dict(kwargs, nrows=1 << 63)
   return _ReadFromPandas(
       pd.read_json,
       path,