Skip to content

Commit

Permalink
assume any file >1MB is a binary file
Browse files Browse the repository at this point in the history
  • Loading branch information
bbean23 committed Aug 23, 2024
1 parent 4a31d75 commit 1e17ec1
Showing 1 changed file with 5 additions and 2 deletions.
7 changes: 5 additions & 2 deletions contrib/scripts/sensitive_strings.py
Original file line number Diff line number Diff line change
Expand Up @@ -109,8 +109,6 @@ def _is_binary_file(self, file_path: str, file_name_ext: str):
ext = ext.lower()
if ext == ".ipynb":
is_binary_file = True
elif ext == ".h5":
is_binary_file = True
elif self._is_img_ext(ext):
if ext in self._text_file_extensions:
is_binary_file = False
Expand All @@ -120,6 +118,11 @@ def _is_binary_file(self, file_path: str, file_name_ext: str):
is_binary_file = False
else:
is_binary_file = True
elif ft.file_size(ft.join(file_path, file_name_ext)) > 1e6:
# assume any file > 1MB is a binary file, in order to prevent
# sensitive_strings from taking hours to check these files
# needlessly
is_binary_file = True
if not is_binary_file:
# attempt to parse the file as a text file
try:
Expand Down

0 comments on commit 1e17ec1

Please sign in to comment.