def file(input_file, light=False):
    util.create_dir(os.path.join(CONF_DIR, "colorschemes/light/"))
    util.create_dir(os.path.join(CONF_DIR, "colorschemes/dark/"))
    theme_name = ".".join((input_file, "json"))
    bri = "light" if light else "dark"
    user_theme_file = os.path.join(CONF_DIR, "colorschemes", bri, theme_name)
    theme_file = os.path.join(MODULE_DIR, "colorschemes", bri, theme_name)
    if input_file in ("random", "random_dark"):
        theme_file = get_random_theme()
    elif input_file == "random_light":
        theme_file = get_random_theme(light)
    elif input_file == "random_user":
        theme_file = get_random_theme_user()
    elif os.path.isfile(user_theme_file):
        theme_file = user_theme_file
    elif os.path.isfile(input_file):
        theme_file = input_file
    if os.path.isfile(theme_file):
        logging.info("Set theme to \033[1;37m%s\033[0m.",
                     os.path.basename(theme_file))
        util.save_file(os.path.basename(theme_file),
                       os.path.join(CACHE_DIR, "last_used_theme"))
        return parse(theme_file)
    logging.error("No %s colorscheme file found.", bri)
    logging.error("Try adding   '-l' to set light themes.")
    logging.error("Try removing '-l' to set dark themes.")
    sys.exit(1)
file(input_file='tests/test_files/test_file.json', light=False)


user_theme_file = os.path.join(CONF_DIR, "colorschemes", bri, theme_name)


def get_random_image(img_dir, recursive):
    if recursive:
        images, current_wall = get_image_dir_recursive(img_dir)
    else:
        images, current_wall = get_image_dir(img_dir)
    if len(images) > 2 and current_wall in images:
        images.remove(current_wall)
    elif not images:
        logging.error("No images found in directory.")
        sys.exit(1)
    random.shuffle(images)
    return os.path.join(img_dir if not recursive else "", images[0])
get_random_image(img_dir='tests/test_files', recursive=False)


def get_image_dir(img_dir):
    current_wall = wallpaper.get()
    current_wall = os.path.basename(current_wall)
    file_types = (".png", ".jpg", ".jpeg", ".jpe", ".gif")
    return [img.name for img in os.scandir(img_dir)
            if img.name.lower().endswith(file_types)], current_wall
get_image_dir(img_dir='tests/test_files')


def merge_args_into_config(args, config_params):
  final_args = copy.deepcopy(config_params)
  config_keys = config_params.keys()
  for key, value in args.items():
    if key in config_keys:
      if any([isinstance(value, t) for t in [str, bool, float, int]]):
        final_args[key] = value
      elif isinstance(value, list):
        final_args[key] = value + config_params[key]
      elif isinstance(value, dict):
        final_args[key].update(**value)
    else:
      final_args[key] = value
  return final_args
merge_args_into_config(args={'input_folder': 'foo/bar', 'resize_images': False, 'im_size': 500, 'compress_pdf': False, 'pdf_im_resolution': 500, 'images_allowlist': {'path1/': 1000}, 'commands_to_delete': ['\\todo1'], 'use_external_tikz': 'foo/bar/tikz'}, config_params={'input_folder': 'foo_/bar_', 'resize_images': True, 'im_size': 1000, 'compress_pdf': True, 'pdf_im_resolution': 1000, 'images_allowlist': {'path2/': 1000}, 'commands_to_delete': ['\\todo2'], 'use_external_tikz': 'foo_/bar_/tikz_'})

{'input_folder': 'foo/bar', 'resize_images': True, 'im_size': 1000, 'compress_pdf': True, 'pdf_im_resolution': 1000, 'images_allowlist': {'path2/': 1000}, 'commands_to_delete': ['\\todo2'], 'use_external_tikz': 'foo_/bar_/tikz_'}

{'input_folder': 'foo/bar', 'resize_images': False, 'im_size': 1000, 'compress_pdf': True, 'pdf_im_resolution': 1000, 'images_allowlist': {'path2/': 1000}, 'commands_to_delete': ['\\todo2'], 'use_external_tikz': 'foo_/bar_/tikz_'}

{'input_folder': 'foo/bar', 'resize_images': False, 'im_size': 500, 'compress_pdf': True, 'pdf_im_resolution': 1000, 'images_allowlist': {'path2/': 1000}, 'commands_to_delete': ['\\todo2'], 'use_external_tikz': 'foo_/bar_/tikz_'}

{'input_folder': 'foo/bar', 'resize_images': False, 'im_size': 500, 'compress_pdf': False, 'pdf_im_resolution': 1000, 'images_allowlist': {'path2/': 1000}, 'commands_to_delete': ['\\todo2'], 'use_external_tikz': 'foo_/bar_/tikz_'}

{'input_folder': 'foo/bar', 'resize_images': False, 'im_size': 500, 'compress_pdf': False, 'pdf_im_resolution': 500, 'images_allowlist': {'path2/': 1000}, 'commands_to_delete': ['\\todo2'], 'use_external_tikz': 'foo_/bar_/tikz_'}

{'input_folder': 'foo/bar', 'resize_images': False, 'im_size': 500, 'compress_pdf': False, 'pdf_im_resolution': 500, 'images_allowlist': {'path2/': 1000, 'path1/': 1000}, 'commands_to_delete': ['\\todo2'], 'use_external_tikz': 'foo_/bar_/tikz_'}

{'input_folder': 'foo/bar', 'resize_images': False, 'im_size': 500, 'compress_pdf': False, 'pdf_im_resolution': 500, 'images_allowlist': {'path2/': 1000, 'path1/': 1000}, 'commands_to_delete': ['\\todo1', '\\todo2'], 'use_external_tikz': 'foo_/bar_/tikz_'}

{'input_folder': 'foo/bar', 'resize_images': False, 'im_size': 500, 'compress_pdf': False, 'pdf_im_resolution': 500, 'images_allowlist': {'path2/': 1000, 'path1/': 1000}, 'commands_to_delete': ['\\todo1', '\\todo2'], 'use_external_tikz': 'foo/bar/tikz'}

dict_keys(['input_folder', 'resize_images', 'im_size', 'compress_pdf', 'pdf_im_resolution', 'images_allowlist', 'commands_to_delete', 'use_external_tikz'])


dict_keys(['input_folder', 'resize_images', 'im_size', 'compress_pdf', 'pdf_im_resolution', 'images_allowlist', 'commands_to_delete', 'use_external_tikz'])


final_args[key] = value + config_params[key]

{'input_folder': 'foo/bar', 'resize_images': False, 'im_size': 500, 'compress_pdf': False, 'pdf_im_resolution': 500, 'images_allowlist': {'path2/': 1000, 'path1/': 1000}, 'commands_to_delete': ['\\todo1', '\\todo2'], 'use_external_tikz': 'foo_/bar_/tikz_'}


def _remove_command(text, command, keep_text=False):
  base_pattern = r'\\' + command + r'\{((?:[^{}]+|\{(?1)\})*)\}'
  while True:
    all_substitutions = []
    has_match = False
    for match in regex.finditer(base_pattern, text):
      has_match = True
      new_substring = (
          ''
          if not keep_text
          else text[match.span()[0] + len(command) + 2 : match.span()[1] - 1]
      )
      if match.span()[1] < len(text):
        next_newline = text[match.span()[1] :].find('\n')
        if next_newline != -1:
          text_until_newline = text[
              match.span()[1] : match.span()[1] + next_newline
          ]
          if (
              not text_until_newline or text_until_newline.isspace()
          ) and not keep_text:
            new_substring = '%'
      all_substitutions.append(
          (match.span()[0], match.span()[1], new_substring)
      )
    for start, end, new_substring in reversed(all_substitutions):
      text = text[:start] + new_substring + text[end:]
    if not keep_text or not has_match:
      break
  return text
_remove_command(text='A\\todo{B\nC}D\nE\n\\end{document}', command='todo', keep_text=False)

'A\\todo{B\nC}D\nE\n\\end{document}'

'A\\todo{B\nC}D\nE\n\\end{document}'

'AD\nE\n\\end{document}'

<regex.Match object; span=(1, 11), match='\\todo{B\nC}'>


'A\\todo{B\nC}D\nE\n\\end{document}'

'A\\todo{B\nC}D\nE\n\\end{document}'

'AD\nE\n\\end{document}'


def _remove_iffalse_block(text):
  p = regex.compile(r'\\if\s*(\w+)|\\fi(?!\w)')
  level = -1
  positions_to_delete = []
  start, end = 0, 0
  for m in p.finditer(text):
    if (
        m.group().replace(' ', '') == r'\iffalse'
        or m.group().replace(' ', '') == r'\if0'
    ) and level == -1:
      level += 1
      start = m.start()
    elif m.group().startswith(r'\if') and level >= 0:
      level += 1
    elif m.group() == r'\fi' and level >= 0:
      if level == 0:
        end = m.end()
        positions_to_delete.append((start, end))
      level -= 1
    else:
      pass
  for start, end in reversed(positions_to_delete):
    if end < len(text) and text[end].isspace():
      end_to_del = end + 1
    else:
      end_to_del = end
    text = text[:start] + text[end_to_del:]
  return text
_remove_iffalse_block(text='\\newcommand\\figref[1]{Figure~\\ref{fig:\\

'\\newcommand\\figref[1]{Figure~\\ref{fig:\\#1}}'


def setUp(self):
    super(IntegrationTests, self).setUp()
    self.out_path = 'tex_arXiv'
setUp(self=<arxiv_latex_cleaner_test.IntegrationTests testMethod=test_complete_from_dir>, __class__=<class 'arxiv_latex_cleaner_test.IntegrationTests'>, self._cleanups=[], self._outcome=<unittest.case._Outcome object at 0x7f1855667a90>, self._subtest=None, self._testMethodDoc="test_complete_from_dir(input_dir='tex')", self._testMethodName='test_complete_from_dir', self._type_equality_funcs={<class 'dict'>: 'assertDictEqual', <class 'list'>: 'assertListEqual', <class 'tuple'>: 'assertTupleEqual', <class 'set'>: 'assertSetEqual', <class 'frozenset'>: 'assertSetEqual', <class 'str'>: 'assertMultiLineEqual'}, self.test_complete_from_dir=<bound method IntegrationTests.test_complete of <arxiv_latex_cleaner_test.IntegrationTests testMethod=test_complete_from_dir>>)

[(<bound method ExitStack.close of <contextlib.ExitStack object at 0x7f1855667d00>>, (), {})]


def test_complete(self, input_dir):
    out_path_true = 'tex_arXiv_true'
    if path.isdir(self.out_path):
      raise RuntimeError(
          'The folder {:s} should not exist.'.format(self.out_path)
      )
    arxiv_latex_cleaner.run_arxiv_cleaner({
        'input_folder': input_dir,
        'images_allowlist': {
            'images/im2_included.jpg': 200,
            'images/im3_included.png': 400,
        },
        'resize_images': True,
        'im_size': 100,
        'compress_pdf': False,
        'pdf_im_resolution': 500,
        'commands_to_delete': ['mytodo'],
        'commands_only_to_delete': ['red'],
        'environments_to_delete': ['mynote'],
        'use_external_tikz': 'ext_tikz',
        'keep_bib': False,
    })
    out_files = set(arxiv_latex_cleaner._list_all_files(self.out_path))
    out_files_true = set(arxiv_latex_cleaner._list_all_files(out_path_true))
    self.assertEqual(out_files, out_files_true)
    for f1 in out_files:
      self._compare_files(
          path.join(self.out_path, f1), path.join(out_path_true, f1)
      )
test_complete(self=<arxiv_latex_cleaner_test.IntegrationTests testMethod=test_complete_from_dir>, input_dir='tex', self._cleanups=[(<bound method ExitStack.close of <contextlib.ExitStack object at 0x7f1855667d00>>, (), {})], self._exit_stack=<contextlib.ExitStack object at 0x7f1855667d00>, self._outcome=<unittest.case._Outcome object at 0x7f1855667a90>, self._subtest=None, self._testMethodDoc="test_complete_from_dir(input_dir='tex')", self._testMethodName='test_complete_from_dir', self._type_equality_funcs={<class 'dict'>: 'assertDictEqual', <class 'list'>: 'assertListEqual', <class 'tuple'>: 'assertTupleEqual', <class 'set'>: 'assertSetEqual', <class 'frozenset'>: 'assertSetEqual', <class 'str'>: 'assertMultiLineEqual'}, self.out_path='tex_arXiv', self.test_complete_from_dir=<bound method IntegrationTests.test_complete of <arxiv_latex_cleaner_test.IntegrationTests testMethod=test_complete_from_dir>>)

{'ext_tikz/test1.pdf', 'figures/figure_included.tex', 'images/im3_included.png', 'ext_tikz/test2.pdf', 'figures/data_included.txt', 'figures/figure_included.tikz', 'main.tex', 'images/im4_included.png', 'images/im1_included.png', 'main.bbl', 'images/include/images/im3_included.png', 'images/im5_included.jpg', 'images/im2_included.jpg'}


def run_arxiv_cleaner(parameters):
  files_to_delete = [
      r'\.aux$',
      r'\.sh$',
      r'\.blg$',
      r'\.brf$',
      r'\.log$',
      r'\.out$',
      r'\.ps$',
      r'\.dvi$',
      r'\.synctex.gz$',
      '~$',
      r'\.backup$',
      r'\.gitignore$',
      r'\.DS_Store$',
      r'\.svg$',
      r'^\.idea',
      r'\.dpth$',
      r'\.md5$',
      r'\.dep$',
      r'\.auxlock$',
      r'\.fls$',
      r'\.fdb_latexmk$',
  ]
  if not parameters['keep_bib']:
    files_to_delete.append(r'\.bib$')
  parameters.update({
      'to_delete': files_to_delete,
      'figures_to_copy_if_referenced': [
          r'\.png$',
          r'\.jpg$',
          r'\.jpeg$',
          r'\.pdf$',
      ],
  })
  logging.info('Collecting file structure.')
  parameters['output_folder'] = _create_out_folder(parameters['input_folder'])
  from_zip = parameters['input_folder'].endswith('.zip')
  tempdir_context = (
      tempfile.TemporaryDirectory() if from_zip else contextlib.suppress()
  )
  with tempdir_context as tempdir:
    if from_zip:
      logging.info('Unzipping input folder.')
      shutil.unpack_archive(parameters['input_folder'], tempdir)
      parameters['input_folder'] = tempdir
    splits = _split_all_files(parameters)
    logging.info('Reading all tex files')
    tex_contents = _read_all_tex_contents(
        splits['tex_in_root'] + splits['tex_not_in_root'], parameters
    )
    for tex_file in tex_contents:
      logging.info('Removing comments in file %s.', tex_file)
      tex_contents[tex_file] = _remove_comments_and_commands_to_delete(
          tex_contents[tex_file], parameters
      )
    for tex_file in tex_contents:
      logging.info('Replacing \\includesvg calls in file %s.', tex_file)
      tex_contents[tex_file] = _replace_includesvg(
          tex_contents[tex_file], splits['svg_inkscape']
      )
    for tex_file in tex_contents:
      logging.info('Replacing Tikz Pictures in file %s.', tex_file)
      content = _replace_tikzpictures(
          tex_contents[tex_file], splits['external_tikz_figures']
      )
      tex_contents[tex_file] = content.split('\n')
    _keep_only_referenced_tex(tex_contents, splits)
    _add_root_tex_files(splits)
    for tex_file in splits['tex_to_copy']:
      logging.info('Replacing patterns in file %s.', tex_file)
      content = '\n'.join(tex_contents[tex_file])
      content = _find_and_replace_patterns(
          content, parameters.get('patterns_and_insertions', list())
      )
      tex_contents[tex_file] = content
      new_path = os.path.join(parameters['output_folder'], tex_file)
      logging.info('Writing modified contents to %s.', new_path)
      _write_file_content(
          content,
          new_path,
      )
    full_content = '\n'.join(
        ''.join(tex_contents[fn]) for fn in splits['tex_to_copy']
    )
    _copy_only_referenced_non_tex_not_in_root(parameters, full_content, splits)
    for non_tex_file in splits['non_tex_in_root']:
      logging.info('Copying non-tex file %s.', non_tex_file)
      _copy_file(non_tex_file, parameters)
    _resize_and_copy_figures_if_referenced(parameters, full_content, splits)
    logging.info('Outputs written to %s', parameters['output_folder'])
run_arxiv_cleaner(parameters={'input_folder': 'tex', 'images_allowlist': {'images/im2_included.jpg': 200, 'images/im3_included.png': 400}, 'resize_images': True, 'im_size': 100, 'compress_pdf': False, 'pdf_im_resolution': 500, 'commands_to_delete': ['mytodo'], 'commands_only_to_delete': ['red'], 'environments_to_delete': ['mynote'], 'use_external_tikz': 'ext_tikz', 'keep_bib': False})

{'input_folder': 'tex', 'images_allowlist': {'images/im2_included.jpg': 200, 'images/im3_included.png': 400}, 'resize_images': True, 'im_size': 100, 'compress_pdf': False, 'pdf_im_resolution': 500, 'commands_to_delete': ['mytodo'], 'commands_only_to_delete': ['red'], 'environments_to_delete': ['mynote'], 'use_external_tikz': 'ext_tikz', 'keep_bib': False, 'to_delete': ['\\.aux$', '\\.sh$', '\\.blg$', '\\.brf$', '\\.log$', '\\.out$', '\\.ps$', '\\.dvi$', '\\.synctex.gz$', '~$', '\\.backup$', '\\.gitignore$', '\\.DS_Store$', '\\.svg$', '^\\.idea', '\\.dpth$', '\\.md5$', '\\.dep$', '\\.auxlock$', '\\.fls$', '\\.fdb_latexmk$', '\\.bib$'], 'figures_to_copy_if_referenced': ['\\.png$', '\\.jpg$', '\\.jpeg$', '\\.pdf$']}

{'input_folder': 'tex', 'images_allowlist': {'images/im2_included.jpg': 200, 'images/im3_included.png': 400}, 'resize_images': True, 'im_size': 100, 'compress_pdf': False, 'pdf_im_resolution': 500, 'commands_to_delete': ['mytodo'], 'commands_only_to_delete': ['red'], 'environments_to_delete': ['mynote'], 'use_external_tikz': 'ext_tikz', 'keep_bib': False, 'to_delete': ['\\.aux$', '\\.sh$', '\\.blg$', '\\.brf$', '\\.log$', '\\.out$', '\\.ps$', '\\.dvi$', '\\.synctex.gz$', '~$', '\\.backup$', '\\.gitignore$', '\\.DS_Store$', '\\.svg$', '^\\.idea', '\\.dpth$', '\\.md5$', '\\.dep$', '\\.auxlock$', '\\.fls$', '\\.fdb_latexmk$', '\\.bib$'], 'figures_to_copy_if_referenced': ['\\.png$', '\\.jpg$', '\\.jpeg$', '\\.pdf$'], 'output_folder': '/local/rcs/XXX/code/pytrace-collector/logs/self_collected/tried/google-research+arxiv-latex-cleaner/google-research+arxiv-latex-cleaner/tex_arXiv'}

{'all': ['main.bib', 'main.bbl', 'main.tex', 'main.aux', 'ext_tikz/test2.pdf', 'ext_tikz/test1.pdf', 'ext_tikz/figure_not_included.pdf', 'figures/data_not_included.txt', 'figures/figure_not_included.tex', 'figures/figure_not_included_2.tex', 'figures/figure_included.tikz', 'figures/figure_included.tex', 'figures/data_included.txt', 'not_included/figures/data_included.txt', 'images/im4_included.png', 'images/im1.png', 'images/im4_not_included.png', 'images/im3_included.png', 'images/im2_included.jpg', 'images/im5_not_included.jpg', 'images/im5_included.jpg', 'images/im1_included.png', 'images/im_not_included.png', 'images/include/images/im3_included.png'], 'in_root': ['main.bib', 'main.bbl', 'main.tex', 'main.aux'], 'not_in_root': ['ext_tikz/test2.pdf', 'ext_tikz/test1.pdf', 'ext_tikz/figure_not_included.pdf', 'figures/data_not_included.txt', 'figures/figure_not_included.tex', 'figures/figure_not_included_2.tex', 'figures/figure_included.tikz', 'figures/figure_included.tex', 'figures/data_included.txt', 'not_included/figures/data_included.txt', 'images/im4_included.png', 'images/im1.png', 'images/im4_not_included.png', 'images/im3_included.png', 'images/im2_included.jpg', 'images/im5_not_included.jpg', 'images/im5_included.jpg', 'images/im1_included.png', 'images/im_not_included.png', 'images/include/images/im3_included.png'], 'to_copy_in_root': ['main.bbl', 'main.tex'], 'to_copy_not_in_root': ['figures/data_not_included.txt', 'figures/figure_not_included.tex', 'figures/figure_not_included_2.tex', 'figures/figure_included.tikz', 'figures/figure_included.tex', 'figures/data_included.txt', 'not_included/figures/data_included.txt'], 'figures': ['ext_tikz/test2.pdf', 'ext_tikz/test1.pdf', 'ext_tikz/figure_not_included.pdf', 'images/im4_included.png', 'images/im1.png', 'images/im4_not_included.png', 'images/im3_included.png', 'images/im2_included.jpg', 'images/im5_not_included.jpg', 'images/im5_included.jpg', 'images/im1_included.png', 'images/im_not_included.png', 'images/include/images/im3_included.png'], 'tex_in_root': ['main.tex'], 'tex_not_in_root': ['figures/figure_not_included.tex', 'figures/figure_not_included_2.tex', 'figures/figure_included.tikz', 'figures/figure_included.tex'], 'non_tex_in_root': ['main.bbl'], 'non_tex_not_in_root': ['figures/data_not_included.txt', 'figures/data_included.txt', 'not_included/figures/data_included.txt'], 'external_tikz_figures': ['ext_tikz/test2.pdf', 'ext_tikz/test1.pdf', 'ext_tikz/figure_not_included.pdf'], 'svg_inkscape': []}

{'all': ['main.bib', 'main.bbl', 'main.tex', 'main.aux', 'ext_tikz/test2.pdf', 'ext_tikz/test1.pdf', 'ext_tikz/figure_not_included.pdf', 'figures/data_not_included.txt', 'figures/figure_not_included.tex', 'figures/figure_not_included_2.tex', 'figures/figure_included.tikz', 'figures/figure_included.tex', 'figures/data_included.txt', 'not_included/figures/data_included.txt', 'images/im4_included.png', 'images/im1.png', 'images/im4_not_included.png', 'images/im3_included.png', 'images/im2_included.jpg', 'images/im5_not_included.jpg', 'images/im5_included.jpg', 'images/im1_included.png', 'images/im_not_included.png', 'images/include/images/im3_included.png'], 'in_root': ['main.bib', 'main.bbl', 'main.tex', 'main.aux'], 'not_in_root': ['ext_tikz/test2.pdf', 'ext_tikz/test1.pdf', 'ext_tikz/figure_not_included.pdf', 'figures/data_not_included.txt', 'figures/figure_not_included.tex', 'figures/figure_not_included_2.tex', 'figures/figure_included.tikz', 'figures/figure_included.tex', 'figures/data_included.txt', 'not_included/figures/data_included.txt', 'images/im4_included.png', 'images/im1.png', 'images/im4_not_included.png', 'images/im3_included.png', 'images/im2_included.jpg', 'images/im5_not_included.jpg', 'images/im5_included.jpg', 'images/im1_included.png', 'images/im_not_included.png', 'images/include/images/im3_included.png'], 'to_copy_in_root': ['main.bbl', 'main.tex'], 'to_copy_not_in_root': ['figures/data_not_included.txt', 'figures/figure_not_included.tex', 'figures/figure_not_included_2.tex', 'figures/figure_included.tikz', 'figures/figure_included.tex', 'figures/data_included.txt', 'not_included/figures/data_included.txt'], 'figures': ['ext_tikz/test2.pdf', 'ext_tikz/test1.pdf', 'ext_tikz/figure_not_included.pdf', 'images/im4_included.png', 'images/im1.png', 'images/im4_not_included.png', 'images/im3_included.png', 'images/im2_included.jpg', 'images/im5_not_included.jpg', 'images/im5_included.jpg', 'images/im1_included.png', 'images/im_not_included.png', 'images/include/images/im3_included.png'], 'tex_in_root': ['main.tex'], 'tex_not_in_root': ['figures/figure_not_included.tex', 'figures/figure_not_included_2.tex', 'figures/figure_included.tikz', 'figures/figure_included.tex'], 'non_tex_in_root': ['main.bbl'], 'non_tex_not_in_root': ['figures/data_not_included.txt', 'figures/data_included.txt', 'not_included/figures/data_included.txt'], 'external_tikz_figures': ['ext_tikz/test2.pdf', 'ext_tikz/test1.pdf', 'ext_tikz/figure_not_included.pdf'], 'svg_inkscape': [], 'tex_to_copy': ['figures/figure_included.tex', 'figures/figure_included.tikz', 'main.tex']}

{'main.tex': ['\\begin{document}\n', 'Text\n', '% Whole line comment\n', '\n', 'Text% Inline comment\n', '\\begin{comment}\n', 'This is an environment comment.\n', '\\end{comment}\n', '\n', 'This is a percent \\%.\n', '% Whole line comment without newline\n', '\\includegraphics{images/im1_included.png}\n', '%\\includegraphics{images/im_not_included}\n', '\\includegraphics{images/im3_included.png}\n', '\\includegraphics{%\n', '  images/im4_included.png%\n', '  }\n', '\\includegraphics[width=.5\\linewidth]{%\n', '  images/im5_included.jpg}\n', '%\\includegraphics{%\n', '%  images/im4_not_included.png\n', '%  }\n', '%\\includegraphics[width=.5\\linewidth]{%\n', '%  images/im5_not_included.jpg}\n', '\n', '% test whatever the path satrting with dot works when include graphics\n', '\\includegraphics{./images/im3_included.png}\n', '\n', 'This line should\\mytodo{Do this later} not be separated\n', '\\mytodo{This is a todo command with a nested \\textit{command}.\n', 'Please remember that up to \\texttt{2 levels} of \\textit{nesting} are supported.}\n', 'from this one.\n', '\n', '\\begin{mynote}\n', '  This is a custom environment that could be excluded.\n', '\\end{mynote}\n', '\n', '\\newif\\ifvar\n', '\n', '\\ifvar\n', '\\if    false\n', '\\if false\n', '\\if 0\n', '\\iffalse\n', '\\ifvar\n', 'Text\n', '\\fi\n', '\\fi\n', '\\fi\n', '\\fi\n', '\\fi\n', '\\fi\n', '\n', '\\newcommand{\\red}[1]{{\\color{red} #1}}\n', 'hello test \\red{hello\n', 'test \\red{hello}}\n', 'test\n', '\n', '% content after this line should not be cleaned if \\end{document} is in a comment\n', '\n', '\\input{figures/figure_included.tex}\n', '% \\input{figures/figure_not_included.tex}\n', '\n', '% Test for tikzpicture feature\n', '% should be replaced\n', '\\tikzsetnextfilename{test1}\n', '\\begin{tikzpicture}\n', '    \\node (test) at (0,0) {Test1};\n', '\\end{tikzpicture}\n', '\n', '% should be replaced in included file\n', '\\input{figures/figure_included.tikz}\n', '\n', '% should not be be replaced - no preceding tikzsetnextfilename command\n', '\\begin{tikzpicture}\n', '    \\node (test) at (0,0) {Test3};\n', '\\end{tikzpicture}\n', '\n', '\\tikzsetnextfilename{test_no_match}\n', '\\begin{tikzpicture}\n', '    \\node (test) at (0,0) {Test4};\n', '\\end{tikzpicture}\n', '\n', '\\end{document}\n'], 'figures/figure_not_included.tex': ['\\addplot{figures/data_not_included.txt}\n', '\\input{figures/figure_not_included_2.tex}\n'], 'figures/figure_not_included_2.tex': [], 'figures/figure_included.tikz': ['\ufeff\\tikzsetnextfilename{test2}\n', '\\begin{tikzpicture}\n', '\\node {root}\n', 'child {node {left}}\n', 'child {node {right}\n', 'child {node {child}}\n', 'child {node {child}}\n', '};\n', '\\end{tikzpicture}'], 'figures/figure_included.tex': ['\\includegraphics{images/im2_included.jpg}\n', '\\addplot{figures/data_included.txt}\n']}

{'main.tex': '\\begin{document}\nText\n\nText%\n\n\nThis is a percent \\%.\n\\includegraphics{images/im1_included.png}\n\\includegraphics{images/im3_included.png}\n\\includegraphics{%\n  images/im4_included.png%\n  }\n\\includegraphics[width=.5\\linewidth]{%\n  images/im5_included.jpg}\n\n\\includegraphics{./images/im3_included.png}\n\nThis line should not be separated\n%\nfrom this one.\n\n\n\n\\newif\\ifvar\n\n\\ifvar\n\\fi\n\n\\newcommand{\\red}[1]{{\\color{red} #1}}\nhello test hello\ntest hello\ntest\n\n\n\\input{figures/figure_included.tex}\n\n\\tikzsetnextfilename{test1}\n\\begin{tikzpicture}\n    \\node (test) at (0,0) {Test1};\n\\end{tikzpicture}\n\n\\input{figures/figure_included.tikz}\n\n\\begin{tikzpicture}\n    \\node (test) at (0,0) {Test3};\n\\end{tikzpicture}\n\n\\tikzsetnextfilename{test_no_match}\n\\begin{tikzpicture}\n    \\node (test) at (0,0) {Test4};\n\\end{tikzpicture}\n\n\\end{document}\n', 'figures/figure_not_included.tex': ['\\addplot{figures/data_not_included.txt}\n', '\\input{figures/figure_not_included_2.tex}\n'], 'figures/figure_not_included_2.tex': [], 'figures/figure_included.tikz': ['\ufeff\\tikzsetnextfilename{test2}\n', '\\begin{tikzpicture}\n', '\\node {root}\n', 'child {node {left}}\n', 'child {node {right}\n', 'child {node {child}}\n', 'child {node {child}}\n', '};\n', '\\end{tikzpicture}'], 'figures/figure_included.tex': ['\\includegraphics{images/im2_included.jpg}\n', '\\addplot{figures/data_included.txt}\n']}

{'main.tex': '\\begin{document}\nText\n\nText%\n\n\nThis is a percent \\%.\n\\includegraphics{images/im1_included.png}\n\\includegraphics{images/im3_included.png}\n\\includegraphics{%\n  images/im4_included.png%\n  }\n\\includegraphics[width=.5\\linewidth]{%\n  images/im5_included.jpg}\n\n\\includegraphics{./images/im3_included.png}\n\nThis line should not be separated\n%\nfrom this one.\n\n\n\n\\newif\\ifvar\n\n\\ifvar\n\\fi\n\n\\newcommand{\\red}[1]{{\\color{red} #1}}\nhello test hello\ntest hello\ntest\n\n\n\\input{figures/figure_included.tex}\n\n\\tikzsetnextfilename{test1}\n\\begin{tikzpicture}\n    \\node (test) at (0,0) {Test1};\n\\end{tikzpicture}\n\n\\input{figures/figure_included.tikz}\n\n\\begin{tikzpicture}\n    \\node (test) at (0,0) {Test3};\n\\end{tikzpicture}\n\n\\tikzsetnextfilename{test_no_match}\n\\begin{tikzpicture}\n    \\node (test) at (0,0) {Test4};\n\\end{tikzpicture}\n\n\\end{document}\n', 'figures/figure_not_included.tex': '\\addplot{figures/data_not_included.txt}\n\\input{figures/figure_not_included_2.tex}\n', 'figures/figure_not_included_2.tex': [], 'figures/figure_included.tikz': ['\ufeff\\tikzsetnextfilename{test2}\n', '\\begin{tikzpicture}\n', '\\node {root}\n', 'child {node {left}}\n', 'child {node {right}\n', 'child {node {child}}\n', 'child {node {child}}\n', '};\n', '\\end{tikzpicture}'], 'figures/figure_included.tex': ['\\includegraphics{images/im2_included.jpg}\n', '\\addplot{figures/data_included.txt}\n']}

{'main.tex': '\\begin{document}\nText\n\nText%\n\n\nThis is a percent \\%.\n\\includegraphics{images/im1_included.png}\n\\includegraphics{images/im3_included.png}\n\\includegraphics{%\n  images/im4_included.png%\n  }\n\\includegraphics[width=.5\\linewidth]{%\n  images/im5_included.jpg}\n\n\\includegraphics{./images/im3_included.png}\n\nThis line should not be separated\n%\nfrom this one.\n\n\n\n\\newif\\ifvar\n\n\\ifvar\n\\fi\n\n\\newcommand{\\red}[1]{{\\color{red} #1}}\nhello test hello\ntest hello\ntest\n\n\n\\input{figures/figure_included.tex}\n\n\\tikzsetnextfilename{test1}\n\\begin{tikzpicture}\n    \\node (test) at (0,0) {Test1};\n\\end{tikzpicture}\n\n\\input{figures/figure_included.tikz}\n\n\\begin{tikzpicture}\n    \\node (test) at (0,0) {Test3};\n\\end{tikzpicture}\n\n\\tikzsetnextfilename{test_no_match}\n\\begin{tikzpicture}\n    \\node (test) at (0,0) {Test4};\n\\end{tikzpicture}\n\n\\end{document}\n', 'figures/figure_not_included.tex': '\\addplot{figures/data_not_included.txt}\n\\input{figures/figure_not_included_2.tex}\n', 'figures/figure_not_included_2.tex': '', 'figures/figure_included.tikz': ['\ufeff\\tikzsetnextfilename{test2}\n', '\\begin{tikzpicture}\n', '\\node {root}\n', 'child {node {left}}\n', 'child {node {right}\n', 'child {node {child}}\n', 'child {node {child}}\n', '};\n', '\\end{tikzpicture}'], 'figures/figure_included.tex': ['\\includegraphics{images/im2_included.jpg}\n', '\\addplot{figures/data_included.txt}\n']}

{'main.tex': '\\begin{document}\nText\n\nText%\n\n\nThis is a percent \\%.\n\\includegraphics{images/im1_included.png}\n\\includegraphics{images/im3_included.png}\n\\includegraphics{%\n  images/im4_included.png%\n  }\n\\includegraphics[width=.5\\linewidth]{%\n  images/im5_included.jpg}\n\n\\includegraphics{./images/im3_included.png}\n\nThis line should not be separated\n%\nfrom this one.\n\n\n\n\\newif\\ifvar\n\n\\ifvar\n\\fi\n\n\\newcommand{\\red}[1]{{\\color{red} #1}}\nhello test hello\ntest hello\ntest\n\n\n\\input{figures/figure_included.tex}\n\n\\tikzsetnextfilename{test1}\n\\begin{tikzpicture}\n    \\node (test) at (0,0) {Test1};\n\\end{tikzpicture}\n\n\\input{figures/figure_included.tikz}\n\n\\begin{tikzpicture}\n    \\node (test) at (0,0) {Test3};\n\\end{tikzpicture}\n\n\\tikzsetnextfilename{test_no_match}\n\\begin{tikzpicture}\n    \\node (test) at (0,0) {Test4};\n\\end{tikzpicture}\n\n\\end{document}\n', 'figures/figure_not_included.tex': '\\addplot{figures/data_not_included.txt}\n\\input{figures/figure_not_included_2.tex}\n', 'figures/figure_not_included_2.tex': '', 'figures/figure_included.tikz': '\ufeff\\tikzsetnextfilename{test2}\n\\begin{tikzpicture}\n\\node {root}\nchild {node {left}}\nchild {node {right}\nchild {node {child}}\nchild {node {child}}\n};\n\\end{tikzpicture}\n', 'figures/figure_included.tex': ['\\includegraphics{images/im2_included.jpg}\n', '\\addplot{figures/data_included.txt}\n']}

{'main.tex': '\\begin{document}\nText\n\nText%\n\n\nThis is a percent \\%.\n\\includegraphics{images/im1_included.png}\n\\includegraphics{images/im3_included.png}\n\\includegraphics{%\n  images/im4_included.png%\n  }\n\\includegraphics[width=.5\\linewidth]{%\n  images/im5_included.jpg}\n\n\\includegraphics{./images/im3_included.png}\n\nThis line should not be separated\n%\nfrom this one.\n\n\n\n\\newif\\ifvar\n\n\\ifvar\n\\fi\n\n\\newcommand{\\red}[1]{{\\color{red} #1}}\nhello test hello\ntest hello\ntest\n\n\n\\input{figures/figure_included.tex}\n\n\\tikzsetnextfilename{test1}\n\\begin{tikzpicture}\n    \\node (test) at (0,0) {Test1};\n\\end{tikzpicture}\n\n\\input{figures/figure_included.tikz}\n\n\\begin{tikzpicture}\n    \\node (test) at (0,0) {Test3};\n\\end{tikzpicture}\n\n\\tikzsetnextfilename{test_no_match}\n\\begin{tikzpicture}\n    \\node (test) at (0,0) {Test4};\n\\end{tikzpicture}\n\n\\end{document}\n', 'figures/figure_not_included.tex': '\\addplot{figures/data_not_included.txt}\n\\input{figures/figure_not_included_2.tex}\n', 'figures/figure_not_included_2.tex': '', 'figures/figure_included.tikz': '\ufeff\\tikzsetnextfilename{test2}\n\\begin{tikzpicture}\n\\node {root}\nchild {node {left}}\nchild {node {right}\nchild {node {child}}\nchild {node {child}}\n};\n\\end{tikzpicture}\n', 'figures/figure_included.tex': '\\includegraphics{images/im2_included.jpg}\n\\addplot{figures/data_included.txt}\n'}

{'main.tex': ['\\begin{document}', 'Text', '', 'Text%', '', '', 'This is a percent \\%.', '\\includegraphics{images/im1_included.png}', '\\includegraphics{images/im3_included.png}', '\\includegraphics{%', '  images/im4_included.png%', '  }', '\\includegraphics[width=.5\\linewidth]{%', '  images/im5_included.jpg}', '', '\\includegraphics{./images/im3_included.png}', '', 'This line should not be separated', '%', 'from this one.', '', '', '', '\\newif\\ifvar', '', '\\ifvar', '\\fi', '', '\\newcommand{\\red}[1]{{\\color{red} #1}}', 'hello test hello', 'test hello', 'test', '', '', '\\input{figures/figure_included.tex}', '', '\\includegraphics{ext_tikz/test1.pdf}', '', '\\input{figures/figure_included.tikz}', '', '\\begin{tikzpicture}', '    \\node (test) at (0,0) {Test3};', '\\end{tikzpicture}', '', '\\tikzsetnextfilename{test_no_match}', '\\begin{tikzpicture}', '    \\node (test) at (0,0) {Test4};', '\\end{tikzpicture}', '', '\\end{document}', ''], 'figures/figure_not_included.tex': '\\addplot{figures/data_not_included.txt}\n\\input{figures/figure_not_included_2.tex}\n', 'figures/figure_not_included_2.tex': '', 'figures/figure_included.tikz': '\ufeff\\tikzsetnextfilename{test2}\n\\begin{tikzpicture}\n\\node {root}\nchild {node {left}}\nchild {node {right}\nchild {node {child}}\nchild {node {child}}\n};\n\\end{tikzpicture}\n', 'figures/figure_included.tex': '\\includegraphics{images/im2_included.jpg}\n\\addplot{figures/data_included.txt}\n'}

{'main.tex': ['\\begin{document}', 'Text', '', 'Text%', '', '', 'This is a percent \\%.', '\\includegraphics{images/im1_included.png}', '\\includegraphics{images/im3_included.png}', '\\includegraphics{%', '  images/im4_included.png%', '  }', '\\includegraphics[width=.5\\linewidth]{%', '  images/im5_included.jpg}', '', '\\includegraphics{./images/im3_included.png}', '', 'This line should not be separated', '%', 'from this one.', '', '', '', '\\newif\\ifvar', '', '\\ifvar', '\\fi', '', '\\newcommand{\\red}[1]{{\\color{red} #1}}', 'hello test hello', 'test hello', 'test', '', '', '\\input{figures/figure_included.tex}', '', '\\includegraphics{ext_tikz/test1.pdf}', '', '\\input{figures/figure_included.tikz}', '', '\\begin{tikzpicture}', '    \\node (test) at (0,0) {Test3};', '\\end{tikzpicture}', '', '\\tikzsetnextfilename{test_no_match}', '\\begin{tikzpicture}', '    \\node (test) at (0,0) {Test4};', '\\end{tikzpicture}', '', '\\end{document}', ''], 'figures/figure_not_included.tex': ['\\addplot{figures/data_not_included.txt}', '\\input{figures/figure_not_included_2.tex}', ''], 'figures/figure_not_included_2.tex': '', 'figures/figure_included.tikz': '\ufeff\\tikzsetnextfilename{test2}\n\\begin{tikzpicture}\n\\node {root}\nchild {node {left}}\nchild {node {right}\nchild {node {child}}\nchild {node {child}}\n};\n\\end{tikzpicture}\n', 'figures/figure_included.tex': '\\includegraphics{images/im2_included.jpg}\n\\addplot{figures/data_included.txt}\n'}

{'main.tex': ['\\begin{document}', 'Text', '', 'Text%', '', '', 'This is a percent \\%.', '\\includegraphics{images/im1_included.png}', '\\includegraphics{images/im3_included.png}', '\\includegraphics{%', '  images/im4_included.png%', '  }', '\\includegraphics[width=.5\\linewidth]{%', '  images/im5_included.jpg}', '', '\\includegraphics{./images/im3_included.png}', '', 'This line should not be separated', '%', 'from this one.', '', '', '', '\\newif\\ifvar', '', '\\ifvar', '\\fi', '', '\\newcommand{\\red}[1]{{\\color{red} #1}}', 'hello test hello', 'test hello', 'test', '', '', '\\input{figures/figure_included.tex}', '', '\\includegraphics{ext_tikz/test1.pdf}', '', '\\input{figures/figure_included.tikz}', '', '\\begin{tikzpicture}', '    \\node (test) at (0,0) {Test3};', '\\end{tikzpicture}', '', '\\tikzsetnextfilename{test_no_match}', '\\begin{tikzpicture}', '    \\node (test) at (0,0) {Test4};', '\\end{tikzpicture}', '', '\\end{document}', ''], 'figures/figure_not_included.tex': ['\\addplot{figures/data_not_included.txt}', '\\input{figures/figure_not_included_2.tex}', ''], 'figures/figure_not_included_2.tex': [''], 'figures/figure_included.tikz': '\ufeff\\tikzsetnextfilename{test2}\n\\begin{tikzpicture}\n\\node {root}\nchild {node {left}}\nchild {node {right}\nchild {node {child}}\nchild {node {child}}\n};\n\\end{tikzpicture}\n', 'figures/figure_included.tex': '\\includegraphics{images/im2_included.jpg}\n\\addplot{figures/data_included.txt}\n'}

{'main.tex': ['\\begin{document}', 'Text', '', 'Text%', '', '', 'This is a percent \\%.', '\\includegraphics{images/im1_included.png}', '\\includegraphics{images/im3_included.png}', '\\includegraphics{%', '  images/im4_included.png%', '  }', '\\includegraphics[width=.5\\linewidth]{%', '  images/im5_included.jpg}', '', '\\includegraphics{./images/im3_included.png}', '', 'This line should not be separated', '%', 'from this one.', '', '', '', '\\newif\\ifvar', '', '\\ifvar', '\\fi', '', '\\newcommand{\\red}[1]{{\\color{red} #1}}', 'hello test hello', 'test hello', 'test', '', '', '\\input{figures/figure_included.tex}', '', '\\includegraphics{ext_tikz/test1.pdf}', '', '\\input{figures/figure_included.tikz}', '', '\\begin{tikzpicture}', '    \\node (test) at (0,0) {Test3};', '\\end{tikzpicture}', '', '\\tikzsetnextfilename{test_no_match}', '\\begin{tikzpicture}', '    \\node (test) at (0,0) {Test4};', '\\end{tikzpicture}', '', '\\end{document}', ''], 'figures/figure_not_included.tex': ['\\addplot{figures/data_not_included.txt}', '\\input{figures/figure_not_included_2.tex}', ''], 'figures/figure_not_included_2.tex': [''], 'figures/figure_included.tikz': ['\ufeff\\includegraphics{ext_tikz/test2.pdf}', ''], 'figures/figure_included.tex': '\\includegraphics{images/im2_included.jpg}\n\\addplot{figures/data_included.txt}\n'}

{'main.tex': ['\\begin{document}', 'Text', '', 'Text%', '', '', 'This is a percent \\%.', '\\includegraphics{images/im1_included.png}', '\\includegraphics{images/im3_included.png}', '\\includegraphics{%', '  images/im4_included.png%', '  }', '\\includegraphics[width=.5\\linewidth]{%', '  images/im5_included.jpg}', '', '\\includegraphics{./images/im3_included.png}', '', 'This line should not be separated', '%', 'from this one.', '', '', '', '\\newif\\ifvar', '', '\\ifvar', '\\fi', '', '\\newcommand{\\red}[1]{{\\color{red} #1}}', 'hello test hello', 'test hello', 'test', '', '', '\\input{figures/figure_included.tex}', '', '\\includegraphics{ext_tikz/test1.pdf}', '', '\\input{figures/figure_included.tikz}', '', '\\begin{tikzpicture}', '    \\node (test) at (0,0) {Test3};', '\\end{tikzpicture}', '', '\\tikzsetnextfilename{test_no_match}', '\\begin{tikzpicture}', '    \\node (test) at (0,0) {Test4};', '\\end{tikzpicture}', '', '\\end{document}', ''], 'figures/figure_not_included.tex': ['\\addplot{figures/data_not_included.txt}', '\\input{figures/figure_not_included_2.tex}', ''], 'figures/figure_not_included_2.tex': [''], 'figures/figure_included.tikz': ['\ufeff\\includegraphics{ext_tikz/test2.pdf}', ''], 'figures/figure_included.tex': ['\\includegraphics{images/im2_included.jpg}', '\\addplot{figures/data_included.txt}', '']}

{'main.tex': ['\\begin{document}', 'Text', '', 'Text%', '', '', 'This is a percent \\%.', '\\includegraphics{images/im1_included.png}', '\\includegraphics{images/im3_included.png}', '\\includegraphics{%', '  images/im4_included.png%', '  }', '\\includegraphics[width=.5\\linewidth]{%', '  images/im5_included.jpg}', '', '\\includegraphics{./images/im3_included.png}', '', 'This line should not be separated', '%', 'from this one.', '', '', '', '\\newif\\ifvar', '', '\\ifvar', '\\fi', '', '\\newcommand{\\red}[1]{{\\color{red} #1}}', 'hello test hello', 'test hello', 'test', '', '', '\\input{figures/figure_included.tex}', '', '\\includegraphics{ext_tikz/test1.pdf}', '', '\\input{figures/figure_included.tikz}', '', '\\begin{tikzpicture}', '    \\node (test) at (0,0) {Test3};', '\\end{tikzpicture}', '', '\\tikzsetnextfilename{test_no_match}', '\\begin{tikzpicture}', '    \\node (test) at (0,0) {Test4};', '\\end{tikzpicture}', '', '\\end{document}', ''], 'figures/figure_not_included.tex': ['\\addplot{figures/data_not_included.txt}', '\\input{figures/figure_not_included_2.tex}', ''], 'figures/figure_not_included_2.tex': [''], 'figures/figure_included.tikz': '\ufeff\\includegraphics{ext_tikz/test2.pdf}\n', 'figures/figure_included.tex': '\\includegraphics{images/im2_included.jpg}\n\\addplot{figures/data_included.txt}\n'}

{'main.tex': '\\begin{document}\nText\n\nText%\n\n\nThis is a percent \\%.\n\\includegraphics{images/im1_included.png}\n\\includegraphics{images/im3_included.png}\n\\includegraphics{%\n  images/im4_included.png%\n  }\n\\includegraphics[width=.5\\linewidth]{%\n  images/im5_included.jpg}\n\n\\includegraphics{./images/im3_included.png}\n\nThis line should not be separated\n%\nfrom this one.\n\n\n\n\\newif\\ifvar\n\n\\ifvar\n\\fi\n\n\\newcommand{\\red}[1]{{\\color{red} #1}}\nhello test hello\ntest hello\ntest\n\n\n\\input{figures/figure_included.tex}\n\n\\includegraphics{ext_tikz/test1.pdf}\n\n\\input{figures/figure_included.tikz}\n\n\\begin{tikzpicture}\n    \\node (test) at (0,0) {Test3};\n\\end{tikzpicture}\n\n\\tikzsetnextfilename{test_no_match}\n\\begin{tikzpicture}\n    \\node (test) at (0,0) {Test4};\n\\end{tikzpicture}\n\n\\end{document}\n', 'figures/figure_not_included.tex': ['\\addplot{figures/data_not_included.txt}', '\\input{figures/figure_not_included_2.tex}', ''], 'figures/figure_not_included_2.tex': [''], 'figures/figure_included.tikz': '\ufeff\\includegraphics{ext_tikz/test2.pdf}\n', 'figures/figure_included.tex': '\\includegraphics{images/im2_included.jpg}\n\\addplot{figures/data_included.txt}\n'}

'\\includegraphics{images/im2_included.jpg}\n\\addplot{figures/data_included.txt}\n\n\ufeff\\includegraphics{ext_tikz/test2.pdf}\n\n\\begin{document}\nText\n\nText%\n\n\nThis is a percent \\%.\n\\includegraphics{images/im1_included.png}\n\\includegraphics{images/im3_included.png}\n\\includegraphics{%\n  images/im4_included.png%\n  }\n\\includegraphics[width=.5\\linewidth]{%\n  images/im5_included.jpg}\n\n\\includegraphics{./images/im3_included.png}\n\nThis line should not be separated\n%\nfrom this one.\n\n\n\n\\newif\\ifvar\n\n\\ifvar\n\\fi\n\n\\newcommand{\\red}[1]{{\\color{red} #1}}\nhello test hello\ntest hello\ntest\n\n\n\\input{figures/figure_included.tex}\n\n\\includegraphics{ext_tikz/test1.pdf}\n\n\\input{figures/figure_included.tikz}\n\n\\begin{tikzpicture}\n    \\node (test) at (0,0) {Test3};\n\\end{tikzpicture}\n\n\\tikzsetnextfilename{test_no_match}\n\\begin{tikzpicture}\n    \\node (test) at (0,0) {Test4};\n\\end{tikzpicture}\n\n\\end{document}\n'


def _split_all_files(parameters):
  file_splits = {
      'all': _list_all_files(
          parameters['input_folder'], ignore_dirs=['.git' + os.sep]
      ),
      'in_root': [
          f
          for f in os.listdir(parameters['input_folder'])
          if os.path.isfile(os.path.join(parameters['input_folder'], f))
      ],
  }
  file_splits['not_in_root'] = [
      f for f in file_splits['all'] if f not in file_splits['in_root']
  ]
  file_splits['to_copy_in_root'] = _remove_pattern(
      file_splits['in_root'],
      parameters['to_delete'] + parameters['figures_to_copy_if_referenced'],
  )
  file_splits['to_copy_not_in_root'] = _remove_pattern(
      file_splits['not_in_root'],
      parameters['to_delete'] + parameters['figures_to_copy_if_referenced'],
  )
  file_splits['figures'] = _keep_pattern(
      file_splits['all'], parameters['figures_to_copy_if_referenced']
  )
  file_splits['tex_in_root'] = _keep_pattern(
      file_splits['to_copy_in_root'], ['.tex$', '.tikz$']
  )
  file_splits['tex_not_in_root'] = _keep_pattern(
      file_splits['to_copy_not_in_root'], ['.tex$', '.tikz$']
  )
  file_splits['non_tex_in_root'] = _remove_pattern(
      file_splits['to_copy_in_root'], ['.tex$', '.tikz$']
  )
  file_splits['non_tex_not_in_root'] = _remove_pattern(
      file_splits['to_copy_not_in_root'], ['.tex$', '.tikz$']
  )
  if parameters.get('use_external_tikz', None) is not None:
    file_splits['external_tikz_figures'] = _keep_pattern(
        file_splits['all'], [parameters['use_external_tikz']]
    )
  else:
    file_splits['external_tikz_figures'] = []
  if parameters.get('svg_inkscape', None) is not None:
    file_splits['svg_inkscape'] = _keep_pattern(
        file_splits['all'], [parameters['svg_inkscape']]
    )
  else:
    file_splits['svg_inkscape'] = []
  return file_splits
_split_all_files(parameters={'input_folder': 'tex', 'images_allowlist': {'images/im2_included.jpg': 200, 'images/im3_included.png': 400}, 'resize_images': True, 'im_size': 100, 'compress_pdf': False, 'pdf_im_resolution': 500, 'commands_to_delete': ['mytodo'], 'commands_only_to_delete': ['red'], 'environments_to_delete': ['mynote'], 'use_external_tikz': 'ext_tikz', 'keep_bib': False, 'to_delete': ['\\.aux$', '\\.sh$', '\\.blg$', '\\.brf$', '\\.log$', '\\.out$', '\\.ps$', '\\.dvi$', '\\.synctex.gz$', '~$', '\\.backup$', '\\.gitignore$', '\\.DS_Store$', '\\.svg$', '^\\.idea', '\\.dpth$', '\\.md5$', '\\.dep$', '\\.auxlock$', '\\.fls$', '\\.fdb_latexmk$', '\\.bib$'], 'figures_to_copy_if_referenced': ['\\.png$', '\\.jpg$', '\\.jpeg$', '\\.pdf$'], 'output_folder': '/local/rcs/XXX/code/pytrace-collector/logs/self_collected/tried/google-research+arxiv-latex-cleaner/google-research+arxiv-latex-cleaner/tex_arXiv'})

{'all': ['main.bib', 'main.bbl', 'main.tex', 'main.aux', 'ext_tikz/test2.pdf', 'ext_tikz/test1.pdf', 'ext_tikz/figure_not_included.pdf', 'figures/data_not_included.txt', 'figures/figure_not_included.tex', 'figures/figure_not_included_2.tex', 'figures/figure_included.tikz', 'figures/figure_included.tex', 'figures/data_included.txt', 'not_included/figures/data_included.txt', 'images/im4_included.png', 'images/im1.png', 'images/im4_not_included.png', 'images/im3_included.png', 'images/im2_included.jpg', 'images/im5_not_included.jpg', 'images/im5_included.jpg', 'images/im1_included.png', 'images/im_not_included.png', 'images/include/images/im3_included.png'], 'in_root': ['main.bib', 'main.bbl', 'main.tex', 'main.aux']}

{'all': ['main.bib', 'main.bbl', 'main.tex', 'main.aux', 'ext_tikz/test2.pdf', 'ext_tikz/test1.pdf', 'ext_tikz/figure_not_included.pdf', 'figures/data_not_included.txt', 'figures/figure_not_included.tex', 'figures/figure_not_included_2.tex', 'figures/figure_included.tikz', 'figures/figure_included.tex', 'figures/data_included.txt', 'not_included/figures/data_included.txt', 'images/im4_included.png', 'images/im1.png', 'images/im4_not_included.png', 'images/im3_included.png', 'images/im2_included.jpg', 'images/im5_not_included.jpg', 'images/im5_included.jpg', 'images/im1_included.png', 'images/im_not_included.png', 'images/include/images/im3_included.png'], 'in_root': ['main.bib', 'main.bbl', 'main.tex', 'main.aux'], 'not_in_root': ['ext_tikz/test2.pdf', 'ext_tikz/test1.pdf', 'ext_tikz/figure_not_included.pdf', 'figures/data_not_included.txt', 'figures/figure_not_included.tex', 'figures/figure_not_included_2.tex', 'figures/figure_included.tikz', 'figures/figure_included.tex', 'figures/data_included.txt', 'not_included/figures/data_included.txt', 'images/im4_included.png', 'images/im1.png', 'images/im4_not_included.png', 'images/im3_included.png', 'images/im2_included.jpg', 'images/im5_not_included.jpg', 'images/im5_included.jpg', 'images/im1_included.png', 'images/im_not_included.png', 'images/include/images/im3_included.png']}

{'all': ['main.bib', 'main.bbl', 'main.tex', 'main.aux', 'ext_tikz/test2.pdf', 'ext_tikz/test1.pdf', 'ext_tikz/figure_not_included.pdf', 'figures/data_not_included.txt', 'figures/figure_not_included.tex', 'figures/figure_not_included_2.tex', 'figures/figure_included.tikz', 'figures/figure_included.tex', 'figures/data_included.txt', 'not_included/figures/data_included.txt', 'images/im4_included.png', 'images/im1.png', 'images/im4_not_included.png', 'images/im3_included.png', 'images/im2_included.jpg', 'images/im5_not_included.jpg', 'images/im5_included.jpg', 'images/im1_included.png', 'images/im_not_included.png', 'images/include/images/im3_included.png'], 'in_root': ['main.bib', 'main.bbl', 'main.tex', 'main.aux'], 'not_in_root': ['ext_tikz/test2.pdf', 'ext_tikz/test1.pdf', 'ext_tikz/figure_not_included.pdf', 'figures/data_not_included.txt', 'figures/figure_not_included.tex', 'figures/figure_not_included_2.tex', 'figures/figure_included.tikz', 'figures/figure_included.tex', 'figures/data_included.txt', 'not_included/figures/data_included.txt', 'images/im4_included.png', 'images/im1.png', 'images/im4_not_included.png', 'images/im3_included.png', 'images/im2_included.jpg', 'images/im5_not_included.jpg', 'images/im5_included.jpg', 'images/im1_included.png', 'images/im_not_included.png', 'images/include/images/im3_included.png'], 'to_copy_in_root': ['main.bbl', 'main.tex']}

{'all': ['main.bib', 'main.bbl', 'main.tex', 'main.aux', 'ext_tikz/test2.pdf', 'ext_tikz/test1.pdf', 'ext_tikz/figure_not_included.pdf', 'figures/data_not_included.txt', 'figures/figure_not_included.tex', 'figures/figure_not_included_2.tex', 'figures/figure_included.tikz', 'figures/figure_included.tex', 'figures/data_included.txt', 'not_included/figures/data_included.txt', 'images/im4_included.png', 'images/im1.png', 'images/im4_not_included.png', 'images/im3_included.png', 'images/im2_included.jpg', 'images/im5_not_included.jpg', 'images/im5_included.jpg', 'images/im1_included.png', 'images/im_not_included.png', 'images/include/images/im3_included.png'], 'in_root': ['main.bib', 'main.bbl', 'main.tex', 'main.aux'], 'not_in_root': ['ext_tikz/test2.pdf', 'ext_tikz/test1.pdf', 'ext_tikz/figure_not_included.pdf', 'figures/data_not_included.txt', 'figures/figure_not_included.tex', 'figures/figure_not_included_2.tex', 'figures/figure_included.tikz', 'figures/figure_included.tex', 'figures/data_included.txt', 'not_included/figures/data_included.txt', 'images/im4_included.png', 'images/im1.png', 'images/im4_not_included.png', 'images/im3_included.png', 'images/im2_included.jpg', 'images/im5_not_included.jpg', 'images/im5_included.jpg', 'images/im1_included.png', 'images/im_not_included.png', 'images/include/images/im3_included.png'], 'to_copy_in_root': ['main.bbl', 'main.tex'], 'to_copy_not_in_root': ['figures/data_not_included.txt', 'figures/figure_not_included.tex', 'figures/figure_not_included_2.tex', 'figures/figure_included.tikz', 'figures/figure_included.tex', 'figures/data_included.txt', 'not_included/figures/data_included.txt']}

{'all': ['main.bib', 'main.bbl', 'main.tex', 'main.aux', 'ext_tikz/test2.pdf', 'ext_tikz/test1.pdf', 'ext_tikz/figure_not_included.pdf', 'figures/data_not_included.txt', 'figures/figure_not_included.tex', 'figures/figure_not_included_2.tex', 'figures/figure_included.tikz', 'figures/figure_included.tex', 'figures/data_included.txt', 'not_included/figures/data_included.txt', 'images/im4_included.png', 'images/im1.png', 'images/im4_not_included.png', 'images/im3_included.png', 'images/im2_included.jpg', 'images/im5_not_included.jpg', 'images/im5_included.jpg', 'images/im1_included.png', 'images/im_not_included.png', 'images/include/images/im3_included.png'], 'in_root': ['main.bib', 'main.bbl', 'main.tex', 'main.aux'], 'not_in_root': ['ext_tikz/test2.pdf', 'ext_tikz/test1.pdf', 'ext_tikz/figure_not_included.pdf', 'figures/data_not_included.txt', 'figures/figure_not_included.tex', 'figures/figure_not_included_2.tex', 'figures/figure_included.tikz', 'figures/figure_included.tex', 'figures/data_included.txt', 'not_included/figures/data_included.txt', 'images/im4_included.png', 'images/im1.png', 'images/im4_not_included.png', 'images/im3_included.png', 'images/im2_included.jpg', 'images/im5_not_included.jpg', 'images/im5_included.jpg', 'images/im1_included.png', 'images/im_not_included.png', 'images/include/images/im3_included.png'], 'to_copy_in_root': ['main.bbl', 'main.tex'], 'to_copy_not_in_root': ['figures/data_not_included.txt', 'figures/figure_not_included.tex', 'figures/figure_not_included_2.tex', 'figures/figure_included.tikz', 'figures/figure_included.tex', 'figures/data_included.txt', 'not_included/figures/data_included.txt'], 'figures': ['ext_tikz/test2.pdf', 'ext_tikz/test1.pdf', 'ext_tikz/figure_not_included.pdf', 'images/im4_included.png', 'images/im1.png', 'images/im4_not_included.png', 'images/im3_included.png', 'images/im2_included.jpg', 'images/im5_not_included.jpg', 'images/im5_included.jpg', 'images/im1_included.png', 'images/im_not_included.png', 'images/include/images/im3_included.png']}

{'all': ['main.bib', 'main.bbl', 'main.tex', 'main.aux', 'ext_tikz/test2.pdf', 'ext_tikz/test1.pdf', 'ext_tikz/figure_not_included.pdf', 'figures/data_not_included.txt', 'figures/figure_not_included.tex', 'figures/figure_not_included_2.tex', 'figures/figure_included.tikz', 'figures/figure_included.tex', 'figures/data_included.txt', 'not_included/figures/data_included.txt', 'images/im4_included.png', 'images/im1.png', 'images/im4_not_included.png', 'images/im3_included.png', 'images/im2_included.jpg', 'images/im5_not_included.jpg', 'images/im5_included.jpg', 'images/im1_included.png', 'images/im_not_included.png', 'images/include/images/im3_included.png'], 'in_root': ['main.bib', 'main.bbl', 'main.tex', 'main.aux'], 'not_in_root': ['ext_tikz/test2.pdf', 'ext_tikz/test1.pdf', 'ext_tikz/figure_not_included.pdf', 'figures/data_not_included.txt', 'figures/figure_not_included.tex', 'figures/figure_not_included_2.tex', 'figures/figure_included.tikz', 'figures/figure_included.tex', 'figures/data_included.txt', 'not_included/figures/data_included.txt', 'images/im4_included.png', 'images/im1.png', 'images/im4_not_included.png', 'images/im3_included.png', 'images/im2_included.jpg', 'images/im5_not_included.jpg', 'images/im5_included.jpg', 'images/im1_included.png', 'images/im_not_included.png', 'images/include/images/im3_included.png'], 'to_copy_in_root': ['main.bbl', 'main.tex'], 'to_copy_not_in_root': ['figures/data_not_included.txt', 'figures/figure_not_included.tex', 'figures/figure_not_included_2.tex', 'figures/figure_included.tikz', 'figures/figure_included.tex', 'figures/data_included.txt', 'not_included/figures/data_included.txt'], 'figures': ['ext_tikz/test2.pdf', 'ext_tikz/test1.pdf', 'ext_tikz/figure_not_included.pdf', 'images/im4_included.png', 'images/im1.png', 'images/im4_not_included.png', 'images/im3_included.png', 'images/im2_included.jpg', 'images/im5_not_included.jpg', 'images/im5_included.jpg', 'images/im1_included.png', 'images/im_not_included.png', 'images/include/images/im3_included.png'], 'tex_in_root': ['main.tex']}

{'all': ['main.bib', 'main.bbl', 'main.tex', 'main.aux', 'ext_tikz/test2.pdf', 'ext_tikz/test1.pdf', 'ext_tikz/figure_not_included.pdf', 'figures/data_not_included.txt', 'figures/figure_not_included.tex', 'figures/figure_not_included_2.tex', 'figures/figure_included.tikz', 'figures/figure_included.tex', 'figures/data_included.txt', 'not_included/figures/data_included.txt', 'images/im4_included.png', 'images/im1.png', 'images/im4_not_included.png', 'images/im3_included.png', 'images/im2_included.jpg', 'images/im5_not_included.jpg', 'images/im5_included.jpg', 'images/im1_included.png', 'images/im_not_included.png', 'images/include/images/im3_included.png'], 'in_root': ['main.bib', 'main.bbl', 'main.tex', 'main.aux'], 'not_in_root': ['ext_tikz/test2.pdf', 'ext_tikz/test1.pdf', 'ext_tikz/figure_not_included.pdf', 'figures/data_not_included.txt', 'figures/figure_not_included.tex', 'figures/figure_not_included_2.tex', 'figures/figure_included.tikz', 'figures/figure_included.tex', 'figures/data_included.txt', 'not_included/figures/data_included.txt', 'images/im4_included.png', 'images/im1.png', 'images/im4_not_included.png', 'images/im3_included.png', 'images/im2_included.jpg', 'images/im5_not_included.jpg', 'images/im5_included.jpg', 'images/im1_included.png', 'images/im_not_included.png', 'images/include/images/im3_included.png'], 'to_copy_in_root': ['main.bbl', 'main.tex'], 'to_copy_not_in_root': ['figures/data_not_included.txt', 'figures/figure_not_included.tex', 'figures/figure_not_included_2.tex', 'figures/figure_included.tikz', 'figures/figure_included.tex', 'figures/data_included.txt', 'not_included/figures/data_included.txt'], 'figures': ['ext_tikz/test2.pdf', 'ext_tikz/test1.pdf', 'ext_tikz/figure_not_included.pdf', 'images/im4_included.png', 'images/im1.png', 'images/im4_not_included.png', 'images/im3_included.png', 'images/im2_included.jpg', 'images/im5_not_included.jpg', 'images/im5_included.jpg', 'images/im1_included.png', 'images/im_not_included.png', 'images/include/images/im3_included.png'], 'tex_in_root': ['main.tex'], 'tex_not_in_root': ['figures/figure_not_included.tex', 'figures/figure_not_included_2.tex', 'figures/figure_included.tikz', 'figures/figure_included.tex']}

{'all': ['main.bib', 'main.bbl', 'main.tex', 'main.aux', 'ext_tikz/test2.pdf', 'ext_tikz/test1.pdf', 'ext_tikz/figure_not_included.pdf', 'figures/data_not_included.txt', 'figures/figure_not_included.tex', 'figures/figure_not_included_2.tex', 'figures/figure_included.tikz', 'figures/figure_included.tex', 'figures/data_included.txt', 'not_included/figures/data_included.txt', 'images/im4_included.png', 'images/im1.png', 'images/im4_not_included.png', 'images/im3_included.png', 'images/im2_included.jpg', 'images/im5_not_included.jpg', 'images/im5_included.jpg', 'images/im1_included.png', 'images/im_not_included.png', 'images/include/images/im3_included.png'], 'in_root': ['main.bib', 'main.bbl', 'main.tex', 'main.aux'], 'not_in_root': ['ext_tikz/test2.pdf', 'ext_tikz/test1.pdf', 'ext_tikz/figure_not_included.pdf', 'figures/data_not_included.txt', 'figures/figure_not_included.tex', 'figures/figure_not_included_2.tex', 'figures/figure_included.tikz', 'figures/figure_included.tex', 'figures/data_included.txt', 'not_included/figures/data_included.txt', 'images/im4_included.png', 'images/im1.png', 'images/im4_not_included.png', 'images/im3_included.png', 'images/im2_included.jpg', 'images/im5_not_included.jpg', 'images/im5_included.jpg', 'images/im1_included.png', 'images/im_not_included.png', 'images/include/images/im3_included.png'], 'to_copy_in_root': ['main.bbl', 'main.tex'], 'to_copy_not_in_root': ['figures/data_not_included.txt', 'figures/figure_not_included.tex', 'figures/figure_not_included_2.tex', 'figures/figure_included.tikz', 'figures/figure_included.tex', 'figures/data_included.txt', 'not_included/figures/data_included.txt'], 'figures': ['ext_tikz/test2.pdf', 'ext_tikz/test1.pdf', 'ext_tikz/figure_not_included.pdf', 'images/im4_included.png', 'images/im1.png', 'images/im4_not_included.png', 'images/im3_included.png', 'images/im2_included.jpg', 'images/im5_not_included.jpg', 'images/im5_included.jpg', 'images/im1_included.png', 'images/im_not_included.png', 'images/include/images/im3_included.png'], 'tex_in_root': ['main.tex'], 'tex_not_in_root': ['figures/figure_not_included.tex', 'figures/figure_not_included_2.tex', 'figures/figure_included.tikz', 'figures/figure_included.tex'], 'non_tex_in_root': ['main.bbl']}

{'all': ['main.bib', 'main.bbl', 'main.tex', 'main.aux', 'ext_tikz/test2.pdf', 'ext_tikz/test1.pdf', 'ext_tikz/figure_not_included.pdf', 'figures/data_not_included.txt', 'figures/figure_not_included.tex', 'figures/figure_not_included_2.tex', 'figures/figure_included.tikz', 'figures/figure_included.tex', 'figures/data_included.txt', 'not_included/figures/data_included.txt', 'images/im4_included.png', 'images/im1.png', 'images/im4_not_included.png', 'images/im3_included.png', 'images/im2_included.jpg', 'images/im5_not_included.jpg', 'images/im5_included.jpg', 'images/im1_included.png', 'images/im_not_included.png', 'images/include/images/im3_included.png'], 'in_root': ['main.bib', 'main.bbl', 'main.tex', 'main.aux'], 'not_in_root': ['ext_tikz/test2.pdf', 'ext_tikz/test1.pdf', 'ext_tikz/figure_not_included.pdf', 'figures/data_not_included.txt', 'figures/figure_not_included.tex', 'figures/figure_not_included_2.tex', 'figures/figure_included.tikz', 'figures/figure_included.tex', 'figures/data_included.txt', 'not_included/figures/data_included.txt', 'images/im4_included.png', 'images/im1.png', 'images/im4_not_included.png', 'images/im3_included.png', 'images/im2_included.jpg', 'images/im5_not_included.jpg', 'images/im5_included.jpg', 'images/im1_included.png', 'images/im_not_included.png', 'images/include/images/im3_included.png'], 'to_copy_in_root': ['main.bbl', 'main.tex'], 'to_copy_not_in_root': ['figures/data_not_included.txt', 'figures/figure_not_included.tex', 'figures/figure_not_included_2.tex', 'figures/figure_included.tikz', 'figures/figure_included.tex', 'figures/data_included.txt', 'not_included/figures/data_included.txt'], 'figures': ['ext_tikz/test2.pdf', 'ext_tikz/test1.pdf', 'ext_tikz/figure_not_included.pdf', 'images/im4_included.png', 'images/im1.png', 'images/im4_not_included.png', 'images/im3_included.png', 'images/im2_included.jpg', 'images/im5_not_included.jpg', 'images/im5_included.jpg', 'images/im1_included.png', 'images/im_not_included.png', 'images/include/images/im3_included.png'], 'tex_in_root': ['main.tex'], 'tex_not_in_root': ['figures/figure_not_included.tex', 'figures/figure_not_included_2.tex', 'figures/figure_included.tikz', 'figures/figure_included.tex'], 'non_tex_in_root': ['main.bbl'], 'non_tex_not_in_root': ['figures/data_not_included.txt', 'figures/data_included.txt', 'not_included/figures/data_included.txt']}

{'all': ['main.bib', 'main.bbl', 'main.tex', 'main.aux', 'ext_tikz/test2.pdf', 'ext_tikz/test1.pdf', 'ext_tikz/figure_not_included.pdf', 'figures/data_not_included.txt', 'figures/figure_not_included.tex', 'figures/figure_not_included_2.tex', 'figures/figure_included.tikz', 'figures/figure_included.tex', 'figures/data_included.txt', 'not_included/figures/data_included.txt', 'images/im4_included.png', 'images/im1.png', 'images/im4_not_included.png', 'images/im3_included.png', 'images/im2_included.jpg', 'images/im5_not_included.jpg', 'images/im5_included.jpg', 'images/im1_included.png', 'images/im_not_included.png', 'images/include/images/im3_included.png'], 'in_root': ['main.bib', 'main.bbl', 'main.tex', 'main.aux'], 'not_in_root': ['ext_tikz/test2.pdf', 'ext_tikz/test1.pdf', 'ext_tikz/figure_not_included.pdf', 'figures/data_not_included.txt', 'figures/figure_not_included.tex', 'figures/figure_not_included_2.tex', 'figures/figure_included.tikz', 'figures/figure_included.tex', 'figures/data_included.txt', 'not_included/figures/data_included.txt', 'images/im4_included.png', 'images/im1.png', 'images/im4_not_included.png', 'images/im3_included.png', 'images/im2_included.jpg', 'images/im5_not_included.jpg', 'images/im5_included.jpg', 'images/im1_included.png', 'images/im_not_included.png', 'images/include/images/im3_included.png'], 'to_copy_in_root': ['main.bbl', 'main.tex'], 'to_copy_not_in_root': ['figures/data_not_included.txt', 'figures/figure_not_included.tex', 'figures/figure_not_included_2.tex', 'figures/figure_included.tikz', 'figures/figure_included.tex', 'figures/data_included.txt', 'not_included/figures/data_included.txt'], 'figures': ['ext_tikz/test2.pdf', 'ext_tikz/test1.pdf', 'ext_tikz/figure_not_included.pdf', 'images/im4_included.png', 'images/im1.png', 'images/im4_not_included.png', 'images/im3_included.png', 'images/im2_included.jpg', 'images/im5_not_included.jpg', 'images/im5_included.jpg', 'images/im1_included.png', 'images/im_not_included.png', 'images/include/images/im3_included.png'], 'tex_in_root': ['main.tex'], 'tex_not_in_root': ['figures/figure_not_included.tex', 'figures/figure_not_included_2.tex', 'figures/figure_included.tikz', 'figures/figure_included.tex'], 'non_tex_in_root': ['main.bbl'], 'non_tex_not_in_root': ['figures/data_not_included.txt', 'figures/data_included.txt', 'not_included/figures/data_included.txt'], 'external_tikz_figures': ['ext_tikz/test2.pdf', 'ext_tikz/test1.pdf', 'ext_tikz/figure_not_included.pdf']}

{'all': ['main.bib', 'main.bbl', 'main.tex', 'main.aux', 'ext_tikz/test2.pdf', 'ext_tikz/test1.pdf', 'ext_tikz/figure_not_included.pdf', 'figures/data_not_included.txt', 'figures/figure_not_included.tex', 'figures/figure_not_included_2.tex', 'figures/figure_included.tikz', 'figures/figure_included.tex', 'figures/data_included.txt', 'not_included/figures/data_included.txt', 'images/im4_included.png', 'images/im1.png', 'images/im4_not_included.png', 'images/im3_included.png', 'images/im2_included.jpg', 'images/im5_not_included.jpg', 'images/im5_included.jpg', 'images/im1_included.png', 'images/im_not_included.png', 'images/include/images/im3_included.png'], 'in_root': ['main.bib', 'main.bbl', 'main.tex', 'main.aux'], 'not_in_root': ['ext_tikz/test2.pdf', 'ext_tikz/test1.pdf', 'ext_tikz/figure_not_included.pdf', 'figures/data_not_included.txt', 'figures/figure_not_included.tex', 'figures/figure_not_included_2.tex', 'figures/figure_included.tikz', 'figures/figure_included.tex', 'figures/data_included.txt', 'not_included/figures/data_included.txt', 'images/im4_included.png', 'images/im1.png', 'images/im4_not_included.png', 'images/im3_included.png', 'images/im2_included.jpg', 'images/im5_not_included.jpg', 'images/im5_included.jpg', 'images/im1_included.png', 'images/im_not_included.png', 'images/include/images/im3_included.png'], 'to_copy_in_root': ['main.bbl', 'main.tex'], 'to_copy_not_in_root': ['figures/data_not_included.txt', 'figures/figure_not_included.tex', 'figures/figure_not_included_2.tex', 'figures/figure_included.tikz', 'figures/figure_included.tex', 'figures/data_included.txt', 'not_included/figures/data_included.txt'], 'figures': ['ext_tikz/test2.pdf', 'ext_tikz/test1.pdf', 'ext_tikz/figure_not_included.pdf', 'images/im4_included.png', 'images/im1.png', 'images/im4_not_included.png', 'images/im3_included.png', 'images/im2_included.jpg', 'images/im5_not_included.jpg', 'images/im5_included.jpg', 'images/im1_included.png', 'images/im_not_included.png', 'images/include/images/im3_included.png'], 'tex_in_root': ['main.tex'], 'tex_not_in_root': ['figures/figure_not_included.tex', 'figures/figure_not_included_2.tex', 'figures/figure_included.tikz', 'figures/figure_included.tex'], 'non_tex_in_root': ['main.bbl'], 'non_tex_not_in_root': ['figures/data_not_included.txt', 'figures/data_included.txt', 'not_included/figures/data_included.txt'], 'external_tikz_figures': ['ext_tikz/test2.pdf', 'ext_tikz/test1.pdf', 'ext_tikz/figure_not_included.pdf'], 'svg_inkscape': []}


def _list_all_files(in_folder, ignore_dirs=None):
  if ignore_dirs is None:
    ignore_dirs = []
  to_consider = [
      os.path.join(os.path.relpath(path, in_folder), name)
      if path != in_folder
      else name
      for path, _, files in os.walk(in_folder)
      for name in files
  ]
  return _remove_pattern(to_consider, ignore_dirs)
_list_all_files(in_folder='tex', ignore_dirs=['.git/'])

['main.bib', 'main.bbl', 'main.tex', 'main.aux', 'ext_tikz/test2.pdf', 'ext_tikz/test1.pdf', 'ext_tikz/figure_not_included.pdf', 'figures/data_not_included.txt', 'figures/figure_not_included.tex', 'figures/figure_not_included_2.tex', 'figures/figure_included.tikz', 'figures/figure_included.tex', 'figures/data_included.txt', 'not_included/figures/data_included.txt', 'images/im4_included.png', 'images/im1.png', 'images/im4_not_included.png', 'images/im3_included.png', 'images/im2_included.jpg', 'images/im5_not_included.jpg', 'images/im5_included.jpg', 'images/im1_included.png', 'images/im_not_included.png', 'images/include/images/im3_included.png']


def _read_all_tex_contents(tex_files, parameters):
  contents = {}
  for fn in tex_files:
    contents[fn] = _read_file_content(
        os.path.join(parameters['input_folder'], fn)
    )
  return contents
_read_all_tex_contents(tex_files=['main.tex', 'figures/figure_not_included.tex', 'figures/figure_not_included_2.tex', 'figures/figure_included.tikz', 'figures/figure_included.tex'], parameters={'input_folder': 'tex', 'images_allowlist': {'images/im2_included.jpg': 200, 'images/im3_included.png': 400}, 'resize_images': True, 'im_size': 100, 'compress_pdf': False, 'pdf_im_resolution': 500, 'commands_to_delete': ['mytodo'], 'commands_only_to_delete': ['red'], 'environments_to_delete': ['mynote'], 'use_external_tikz': 'ext_tikz', 'keep_bib': False, 'to_delete': ['\\.aux$', '\\.sh$', '\\.blg$', '\\.brf$', '\\.log$', '\\.out$', '\\.ps$', '\\.dvi$', '\\.synctex.gz$', '~$', '\\.backup$', '\\.gitignore$', '\\.DS_Store$', '\\.svg$', '^\\.idea', '\\.dpth$', '\\.md5$', '\\.dep$', '\\.auxlock$', '\\.fls$', '\\.fdb_latexmk$', '\\.bib$'], 'figures_to_copy_if_referenced': ['\\.png$', '\\.jpg$', '\\.jpeg$', '\\.pdf$'], 'output_folder': '/local/rcs/XXX/code/pytrace-collector/logs/self_collected/tried/google-research+arxiv-latex-cleaner/google-research+arxiv-latex-cleaner/tex_arXiv'})

{'main.tex': ['\\begin{document}\n', 'Text\n', '% Whole line comment\n', '\n', 'Text% Inline comment\n', '\\begin{comment}\n', 'This is an environment comment.\n', '\\end{comment}\n', '\n', 'This is a percent \\%.\n', '% Whole line comment without newline\n', '\\includegraphics{images/im1_included.png}\n', '%\\includegraphics{images/im_not_included}\n', '\\includegraphics{images/im3_included.png}\n', '\\includegraphics{%\n', '  images/im4_included.png%\n', '  }\n', '\\includegraphics[width=.5\\linewidth]{%\n', '  images/im5_included.jpg}\n', '%\\includegraphics{%\n', '%  images/im4_not_included.png\n', '%  }\n', '%\\includegraphics[width=.5\\linewidth]{%\n', '%  images/im5_not_included.jpg}\n', '\n', '% test whatever the path satrting with dot works when include graphics\n', '\\includegraphics{./images/im3_included.png}\n', '\n', 'This line should\\mytodo{Do this later} not be separated\n', '\\mytodo{This is a todo command with a nested \\textit{command}.\n', 'Please remember that up to \\texttt{2 levels} of \\textit{nesting} are supported.}\n', 'from this one.\n', '\n', '\\begin{mynote}\n', '  This is a custom environment that could be excluded.\n', '\\end{mynote}\n', '\n', '\\newif\\ifvar\n', '\n', '\\ifvar\n', '\\if    false\n', '\\if false\n', '\\if 0\n', '\\iffalse\n', '\\ifvar\n', 'Text\n', '\\fi\n', '\\fi\n', '\\fi\n', '\\fi\n', '\\fi\n', '\\fi\n', '\n', '\\newcommand{\\red}[1]{{\\color{red} #1}}\n', 'hello test \\red{hello\n', 'test \\red{hello}}\n', 'test\n', '\n', '% content after this line should not be cleaned if \\end{document} is in a comment\n', '\n', '\\input{figures/figure_included.tex}\n', '% \\input{figures/figure_not_included.tex}\n', '\n', '% Test for tikzpicture feature\n', '% should be replaced\n', '\\tikzsetnextfilename{test1}\n', '\\begin{tikzpicture}\n', '    \\node (test) at (0,0) {Test1};\n', '\\end{tikzpicture}\n', '\n', '% should be replaced in included file\n', '\\input{figures/figure_included.tikz}\n', '\n', '% should not be be replaced - no preceding tikzsetnextfilename command\n', '\\begin{tikzpicture}\n', '    \\node (test) at (0,0) {Test3};\n', '\\end{tikzpicture}\n', '\n', '\\tikzsetnextfilename{test_no_match}\n', '\\begin{tikzpicture}\n', '    \\node (test) at (0,0) {Test4};\n', '\\end{tikzpicture}\n', '\n', '\\end{document}\n']}

{'main.tex': ['\\begin{document}\n', 'Text\n', '% Whole line comment\n', '\n', 'Text% Inline comment\n', '\\begin{comment}\n', 'This is an environment comment.\n', '\\end{comment}\n', '\n', 'This is a percent \\%.\n', '% Whole line comment without newline\n', '\\includegraphics{images/im1_included.png}\n', '%\\includegraphics{images/im_not_included}\n', '\\includegraphics{images/im3_included.png}\n', '\\includegraphics{%\n', '  images/im4_included.png%\n', '  }\n', '\\includegraphics[width=.5\\linewidth]{%\n', '  images/im5_included.jpg}\n', '%\\includegraphics{%\n', '%  images/im4_not_included.png\n', '%  }\n', '%\\includegraphics[width=.5\\linewidth]{%\n', '%  images/im5_not_included.jpg}\n', '\n', '% test whatever the path satrting with dot works when include graphics\n', '\\includegraphics{./images/im3_included.png}\n', '\n', 'This line should\\mytodo{Do this later} not be separated\n', '\\mytodo{This is a todo command with a nested \\textit{command}.\n', 'Please remember that up to \\texttt{2 levels} of \\textit{nesting} are supported.}\n', 'from this one.\n', '\n', '\\begin{mynote}\n', '  This is a custom environment that could be excluded.\n', '\\end{mynote}\n', '\n', '\\newif\\ifvar\n', '\n', '\\ifvar\n', '\\if    false\n', '\\if false\n', '\\if 0\n', '\\iffalse\n', '\\ifvar\n', 'Text\n', '\\fi\n', '\\fi\n', '\\fi\n', '\\fi\n', '\\fi\n', '\\fi\n', '\n', '\\newcommand{\\red}[1]{{\\color{red} #1}}\n', 'hello test \\red{hello\n', 'test \\red{hello}}\n', 'test\n', '\n', '% content after this line should not be cleaned if \\end{document} is in a comment\n', '\n', '\\input{figures/figure_included.tex}\n', '% \\input{figures/figure_not_included.tex}\n', '\n', '% Test for tikzpicture feature\n', '% should be replaced\n', '\\tikzsetnextfilename{test1}\n', '\\begin{tikzpicture}\n', '    \\node (test) at (0,0) {Test1};\n', '\\end{tikzpicture}\n', '\n', '% should be replaced in included file\n', '\\input{figures/figure_included.tikz}\n', '\n', '% should not be be replaced - no preceding tikzsetnextfilename command\n', '\\begin{tikzpicture}\n', '    \\node (test) at (0,0) {Test3};\n', '\\end{tikzpicture}\n', '\n', '\\tikzsetnextfilename{test_no_match}\n', '\\begin{tikzpicture}\n', '    \\node (test) at (0,0) {Test4};\n', '\\end{tikzpicture}\n', '\n', '\\end{document}\n'], 'figures/figure_not_included.tex': ['\\addplot{figures/data_not_included.txt}\n', '\\input{figures/figure_not_included_2.tex}\n']}

{'main.tex': ['\\begin{document}\n', 'Text\n', '% Whole line comment\n', '\n', 'Text% Inline comment\n', '\\begin{comment}\n', 'This is an environment comment.\n', '\\end{comment}\n', '\n', 'This is a percent \\%.\n', '% Whole line comment without newline\n', '\\includegraphics{images/im1_included.png}\n', '%\\includegraphics{images/im_not_included}\n', '\\includegraphics{images/im3_included.png}\n', '\\includegraphics{%\n', '  images/im4_included.png%\n', '  }\n', '\\includegraphics[width=.5\\linewidth]{%\n', '  images/im5_included.jpg}\n', '%\\includegraphics{%\n', '%  images/im4_not_included.png\n', '%  }\n', '%\\includegraphics[width=.5\\linewidth]{%\n', '%  images/im5_not_included.jpg}\n', '\n', '% test whatever the path satrting with dot works when include graphics\n', '\\includegraphics{./images/im3_included.png}\n', '\n', 'This line should\\mytodo{Do this later} not be separated\n', '\\mytodo{This is a todo command with a nested \\textit{command}.\n', 'Please remember that up to \\texttt{2 levels} of \\textit{nesting} are supported.}\n', 'from this one.\n', '\n', '\\begin{mynote}\n', '  This is a custom environment that could be excluded.\n', '\\end{mynote}\n', '\n', '\\newif\\ifvar\n', '\n', '\\ifvar\n', '\\if    false\n', '\\if false\n', '\\if 0\n', '\\iffalse\n', '\\ifvar\n', 'Text\n', '\\fi\n', '\\fi\n', '\\fi\n', '\\fi\n', '\\fi\n', '\\fi\n', '\n', '\\newcommand{\\red}[1]{{\\color{red} #1}}\n', 'hello test \\red{hello\n', 'test \\red{hello}}\n', 'test\n', '\n', '% content after this line should not be cleaned if \\end{document} is in a comment\n', '\n', '\\input{figures/figure_included.tex}\n', '% \\input{figures/figure_not_included.tex}\n', '\n', '% Test for tikzpicture feature\n', '% should be replaced\n', '\\tikzsetnextfilename{test1}\n', '\\begin{tikzpicture}\n', '    \\node (test) at (0,0) {Test1};\n', '\\end{tikzpicture}\n', '\n', '% should be replaced in included file\n', '\\input{figures/figure_included.tikz}\n', '\n', '% should not be be replaced - no preceding tikzsetnextfilename command\n', '\\begin{tikzpicture}\n', '    \\node (test) at (0,0) {Test3};\n', '\\end{tikzpicture}\n', '\n', '\\tikzsetnextfilename{test_no_match}\n', '\\begin{tikzpicture}\n', '    \\node (test) at (0,0) {Test4};\n', '\\end{tikzpicture}\n', '\n', '\\end{document}\n'], 'figures/figure_not_included.tex': ['\\addplot{figures/data_not_included.txt}\n', '\\input{figures/figure_not_included_2.tex}\n'], 'figures/figure_not_included_2.tex': []}

{'main.tex': ['\\begin{document}\n', 'Text\n', '% Whole line comment\n', '\n', 'Text% Inline comment\n', '\\begin{comment}\n', 'This is an environment comment.\n', '\\end{comment}\n', '\n', 'This is a percent \\%.\n', '% Whole line comment without newline\n', '\\includegraphics{images/im1_included.png}\n', '%\\includegraphics{images/im_not_included}\n', '\\includegraphics{images/im3_included.png}\n', '\\includegraphics{%\n', '  images/im4_included.png%\n', '  }\n', '\\includegraphics[width=.5\\linewidth]{%\n', '  images/im5_included.jpg}\n', '%\\includegraphics{%\n', '%  images/im4_not_included.png\n', '%  }\n', '%\\includegraphics[width=.5\\linewidth]{%\n', '%  images/im5_not_included.jpg}\n', '\n', '% test whatever the path satrting with dot works when include graphics\n', '\\includegraphics{./images/im3_included.png}\n', '\n', 'This line should\\mytodo{Do this later} not be separated\n', '\\mytodo{This is a todo command with a nested \\textit{command}.\n', 'Please remember that up to \\texttt{2 levels} of \\textit{nesting} are supported.}\n', 'from this one.\n', '\n', '\\begin{mynote}\n', '  This is a custom environment that could be excluded.\n', '\\end{mynote}\n', '\n', '\\newif\\ifvar\n', '\n', '\\ifvar\n', '\\if    false\n', '\\if false\n', '\\if 0\n', '\\iffalse\n', '\\ifvar\n', 'Text\n', '\\fi\n', '\\fi\n', '\\fi\n', '\\fi\n', '\\fi\n', '\\fi\n', '\n', '\\newcommand{\\red}[1]{{\\color{red} #1}}\n', 'hello test \\red{hello\n', 'test \\red{hello}}\n', 'test\n', '\n', '% content after this line should not be cleaned if \\end{document} is in a comment\n', '\n', '\\input{figures/figure_included.tex}\n', '% \\input{figures/figure_not_included.tex}\n', '\n', '% Test for tikzpicture feature\n', '% should be replaced\n', '\\tikzsetnextfilename{test1}\n', '\\begin{tikzpicture}\n', '    \\node (test) at (0,0) {Test1};\n', '\\end{tikzpicture}\n', '\n', '% should be replaced in included file\n', '\\input{figures/figure_included.tikz}\n', '\n', '% should not be be replaced - no preceding tikzsetnextfilename command\n', '\\begin{tikzpicture}\n', '    \\node (test) at (0,0) {Test3};\n', '\\end{tikzpicture}\n', '\n', '\\tikzsetnextfilename{test_no_match}\n', '\\begin{tikzpicture}\n', '    \\node (test) at (0,0) {Test4};\n', '\\end{tikzpicture}\n', '\n', '\\end{document}\n'], 'figures/figure_not_included.tex': ['\\addplot{figures/data_not_included.txt}\n', '\\input{figures/figure_not_included_2.tex}\n'], 'figures/figure_not_included_2.tex': [], 'figures/figure_included.tikz': ['\ufeff\\tikzsetnextfilename{test2}\n', '\\begin{tikzpicture}\n', '\\node {root}\n', 'child {node {left}}\n', 'child {node {right}\n', 'child {node {child}}\n', 'child {node {child}}\n', '};\n', '\\end{tikzpicture}']}

{'main.tex': ['\\begin{document}\n', 'Text\n', '% Whole line comment\n', '\n', 'Text% Inline comment\n', '\\begin{comment}\n', 'This is an environment comment.\n', '\\end{comment}\n', '\n', 'This is a percent \\%.\n', '% Whole line comment without newline\n', '\\includegraphics{images/im1_included.png}\n', '%\\includegraphics{images/im_not_included}\n', '\\includegraphics{images/im3_included.png}\n', '\\includegraphics{%\n', '  images/im4_included.png%\n', '  }\n', '\\includegraphics[width=.5\\linewidth]{%\n', '  images/im5_included.jpg}\n', '%\\includegraphics{%\n', '%  images/im4_not_included.png\n', '%  }\n', '%\\includegraphics[width=.5\\linewidth]{%\n', '%  images/im5_not_included.jpg}\n', '\n', '% test whatever the path satrting with dot works when include graphics\n', '\\includegraphics{./images/im3_included.png}\n', '\n', 'This line should\\mytodo{Do this later} not be separated\n', '\\mytodo{This is a todo command with a nested \\textit{command}.\n', 'Please remember that up to \\texttt{2 levels} of \\textit{nesting} are supported.}\n', 'from this one.\n', '\n', '\\begin{mynote}\n', '  This is a custom environment that could be excluded.\n', '\\end{mynote}\n', '\n', '\\newif\\ifvar\n', '\n', '\\ifvar\n', '\\if    false\n', '\\if false\n', '\\if 0\n', '\\iffalse\n', '\\ifvar\n', 'Text\n', '\\fi\n', '\\fi\n', '\\fi\n', '\\fi\n', '\\fi\n', '\\fi\n', '\n', '\\newcommand{\\red}[1]{{\\color{red} #1}}\n', 'hello test \\red{hello\n', 'test \\red{hello}}\n', 'test\n', '\n', '% content after this line should not be cleaned if \\end{document} is in a comment\n', '\n', '\\input{figures/figure_included.tex}\n', '% \\input{figures/figure_not_included.tex}\n', '\n', '% Test for tikzpicture feature\n', '% should be replaced\n', '\\tikzsetnextfilename{test1}\n', '\\begin{tikzpicture}\n', '    \\node (test) at (0,0) {Test1};\n', '\\end{tikzpicture}\n', '\n', '% should be replaced in included file\n', '\\input{figures/figure_included.tikz}\n', '\n', '% should not be be replaced - no preceding tikzsetnextfilename command\n', '\\begin{tikzpicture}\n', '    \\node (test) at (0,0) {Test3};\n', '\\end{tikzpicture}\n', '\n', '\\tikzsetnextfilename{test_no_match}\n', '\\begin{tikzpicture}\n', '    \\node (test) at (0,0) {Test4};\n', '\\end{tikzpicture}\n', '\n', '\\end{document}\n'], 'figures/figure_not_included.tex': ['\\addplot{figures/data_not_included.txt}\n', '\\input{figures/figure_not_included_2.tex}\n'], 'figures/figure_not_included_2.tex': [], 'figures/figure_included.tikz': ['\ufeff\\tikzsetnextfilename{test2}\n', '\\begin{tikzpicture}\n', '\\node {root}\n', 'child {node {left}}\n', 'child {node {right}\n', 'child {node {child}}\n', 'child {node {child}}\n', '};\n', '\\end{tikzpicture}'], 'figures/figure_included.tex': ['\\includegraphics{images/im2_included.jpg}\n', '\\addplot{figures/data_included.txt}\n']}


def _read_file_content(filename):
  with open(filename, 'r', encoding='utf-8') as fp:
    lines = fp.readlines()
    lines = _strip_tex_contents(lines, '\\end{document}')
    return lines
_read_file_content(filename='tex/main.tex')

['\\begin{document}\n', 'Text\n', '% Whole line comment\n', '\n', 'Text% Inline comment\n', '\\begin{comment}\n', 'This is an environment comment.\n', '\\end{comment}\n', '\n', 'This is a percent \\%.\n', '% Whole line comment without newline\n', '\\includegraphics{images/im1_included.png}\n', '%\\includegraphics{images/im_not_included}\n', '\\includegraphics{images/im3_included.png}\n', '\\includegraphics{%\n', '  images/im4_included.png%\n', '  }\n', '\\includegraphics[width=.5\\linewidth]{%\n', '  images/im5_included.jpg}\n', '%\\includegraphics{%\n', '%  images/im4_not_included.png\n', '%  }\n', '%\\includegraphics[width=.5\\linewidth]{%\n', '%  images/im5_not_included.jpg}\n', '\n', '% test whatever the path satrting with dot works when include graphics\n', '\\includegraphics{./images/im3_included.png}\n', '\n', 'This line should\\mytodo{Do this later} not be separated\n', '\\mytodo{This is a todo command with a nested \\textit{command}.\n', 'Please remember that up to \\texttt{2 levels} of \\textit{nesting} are supported.}\n', 'from this one.\n', '\n', '\\begin{mynote}\n', '  This is a custom environment that could be excluded.\n', '\\end{mynote}\n', '\n', '\\newif\\ifvar\n', '\n', '\\ifvar\n', '\\if    false\n', '\\if false\n', '\\if 0\n', '\\iffalse\n', '\\ifvar\n', 'Text\n', '\\fi\n', '\\fi\n', '\\fi\n', '\\fi\n', '\\fi\n', '\\fi\n', '\n', '\\newcommand{\\red}[1]{{\\color{red} #1}}\n', 'hello test \\red{hello\n', 'test \\red{hello}}\n', 'test\n', '\n', '% content after this line should not be cleaned if \\end{document} is in a comment\n', '\n', '\\input{figures/figure_included.tex}\n', '% \\input{figures/figure_not_included.tex}\n', '\n', '% Test for tikzpicture feature\n', '% should be replaced\n', '\\tikzsetnextfilename{test1}\n', '\\begin{tikzpicture}\n', '    \\node (test) at (0,0) {Test1};\n', '\\end{tikzpicture}\n', '\n', '% should be replaced in included file\n', '\\input{figures/figure_included.tikz}\n', '\n', '% should not be be replaced - no preceding tikzsetnextfilename command\n', '\\begin{tikzpicture}\n', '    \\node (test) at (0,0) {Test3};\n', '\\end{tikzpicture}\n', '\n', '\\tikzsetnextfilename{test_no_match}\n', '\\begin{tikzpicture}\n', '    \\node (test) at (0,0) {Test4};\n', '\\end{tikzpicture}\n', '\n', '\\end{document}\n', '\n', 'This should be ignored.\n']

<_io.TextIOWrapper name='tex/main.tex' mode='r' encoding='utf-8'>

['\\begin{document}\n', 'Text\n', '% Whole line comment\n', '\n', 'Text% Inline comment\n', '\\begin{comment}\n', 'This is an environment comment.\n', '\\end{comment}\n', '\n', 'This is a percent \\%.\n', '% Whole line comment without newline\n', '\\includegraphics{images/im1_included.png}\n', '%\\includegraphics{images/im_not_included}\n', '\\includegraphics{images/im3_included.png}\n', '\\includegraphics{%\n', '  images/im4_included.png%\n', '  }\n', '\\includegraphics[width=.5\\linewidth]{%\n', '  images/im5_included.jpg}\n', '%\\includegraphics{%\n', '%  images/im4_not_included.png\n', '%  }\n', '%\\includegraphics[width=.5\\linewidth]{%\n', '%  images/im5_not_included.jpg}\n', '\n', '% test whatever the path satrting with dot works when include graphics\n', '\\includegraphics{./images/im3_included.png}\n', '\n', 'This line should\\mytodo{Do this later} not be separated\n', '\\mytodo{This is a todo command with a nested \\textit{command}.\n', 'Please remember that up to \\texttt{2 levels} of \\textit{nesting} are supported.}\n', 'from this one.\n', '\n', '\\begin{mynote}\n', '  This is a custom environment that could be excluded.\n', '\\end{mynote}\n', '\n', '\\newif\\ifvar\n', '\n', '\\ifvar\n', '\\if    false\n', '\\if false\n', '\\if 0\n', '\\iffalse\n', '\\ifvar\n', 'Text\n', '\\fi\n', '\\fi\n', '\\fi\n', '\\fi\n', '\\fi\n', '\\fi\n', '\n', '\\newcommand{\\red}[1]{{\\color{red} #1}}\n', 'hello test \\red{hello\n', 'test \\red{hello}}\n', 'test\n', '\n', '% content after this line should not be cleaned if \\end{document} is in a comment\n', '\n', '\\input{figures/figure_included.tex}\n', '% \\input{figures/figure_not_included.tex}\n', '\n', '% Test for tikzpicture feature\n', '% should be replaced\n', '\\tikzsetnextfilename{test1}\n', '\\begin{tikzpicture}\n', '    \\node (test) at (0,0) {Test1};\n', '\\end{tikzpicture}\n', '\n', '% should be replaced in included file\n', '\\input{figures/figure_included.tikz}\n', '\n', '% should not be be replaced - no preceding tikzsetnextfilename command\n', '\\begin{tikzpicture}\n', '    \\node (test) at (0,0) {Test3};\n', '\\end{tikzpicture}\n', '\n', '\\tikzsetnextfilename{test_no_match}\n', '\\begin{tikzpicture}\n', '    \\node (test) at (0,0) {Test4};\n', '\\end{tikzpicture}\n', '\n', '\\end{document}\n']


lines = _strip_tex_contents(lines, '\\end{document}')

['\\begin{document}\n', 'Text\n', '% Whole line comment\n', '\n', 'Text% Inline comment\n', '\\begin{comment}\n', 'This is an environment comment.\n', '\\end{comment}\n', '\n', 'This is a percent \\%.\n', '% Whole line comment without newline\n', '\\includegraphics{images/im1_included.png}\n', '%\\includegraphics{images/im_not_included}\n', '\\includegraphics{images/im3_included.png}\n', '\\includegraphics{%\n', '  images/im4_included.png%\n', '  }\n', '\\includegraphics[width=.5\\linewidth]{%\n', '  images/im5_included.jpg}\n', '%\\includegraphics{%\n', '%  images/im4_not_included.png\n', '%  }\n', '%\\includegraphics[width=.5\\linewidth]{%\n', '%  images/im5_not_included.jpg}\n', '\n', '% test whatever the path satrting with dot works when include graphics\n', '\\includegraphics{./images/im3_included.png}\n', '\n', 'This line should\\mytodo{Do this later} not be separated\n', '\\mytodo{This is a todo command with a nested \\textit{command}.\n', 'Please remember that up to \\texttt{2 levels} of \\textit{nesting} are supported.}\n', 'from this one.\n', '\n', '\\begin{mynote}\n', '  This is a custom environment that could be excluded.\n', '\\end{mynote}\n', '\n', '\\newif\\ifvar\n', '\n', '\\ifvar\n', '\\if    false\n', '\\if false\n', '\\if 0\n', '\\iffalse\n', '\\ifvar\n', 'Text\n', '\\fi\n', '\\fi\n', '\\fi\n', '\\fi\n', '\\fi\n', '\\fi\n', '\n', '\\newcommand{\\red}[1]{{\\color{red} #1}}\n', 'hello test \\red{hello\n', 'test \\red{hello}}\n', 'test\n', '\n', '% content after this line should not be cleaned if \\end{document} is in a comment\n', '\n', '\\input{figures/figure_included.tex}\n', '% \\input{figures/figure_not_included.tex}\n', '\n', '% Test for tikzpicture feature\n', '% should be replaced\n', '\\tikzsetnextfilename{test1}\n', '\\begin{tikzpicture}\n', '    \\node (test) at (0,0) {Test1};\n', '\\end{tikzpicture}\n', '\n', '% should be replaced in included file\n', '\\input{figures/figure_included.tikz}\n', '\n', '% should not be be replaced - no preceding tikzsetnextfilename command\n', '\\begin{tikzpicture}\n', '    \\node (test) at (0,0) {Test3};\n', '\\end{tikzpicture}\n', '\n', '\\tikzsetnextfilename{test_no_match}\n', '\\begin{tikzpicture}\n', '    \\node (test) at (0,0) {Test4};\n', '\\end{tikzpicture}\n', '\n', '\\end{document}\n']