Moving binary to textual file openings

Author: stamparm

data/txt/sha256sums.txt

@@ -168,28 +168,28 @@ e376093d4f6e42ee38b050af329179df9c1c136b7667b2f1cb559f5d4b69ebd9  lib/controller
 1966ca704961fb987ab757f0a4afddbf841d1a880631b701487c75cef63d60c3  lib/controller/__init__.py
 2a96190ced25d8929861b13866101812fcadf5cac23dd1dd4b29b1a915918769  lib/core/agent.py
 1da4ec9cd9b67c8b54e4a3d314f8237d58778d8f3a00bc26a1e0540294dca30f  lib/core/bigarray.py
-3324bcf3da983e8ca8d34fc1123b32706056e7158aaa2c274fa1560b1f377889  lib/core/common.py
+af24159b8ca5b8fe5e13cdfdedc2a758a2f4883361a601e0a550127cff368b3a  lib/core/common.py
 a6397b10de7ae7c56ed6b0fa3b3c58eb7a9dbede61bf93d786e73258175c981e  lib/core/compat.py
 a9997e97ebe88e0bf7efcf21e878bc5f62c72348e5aba18f64d6861390a4dcf2  lib/core/convert.py
 c03dc585f89642cfd81b087ac2723e3e1bb3bfa8c60e6f5fe58ef3b0113ebfe6  lib/core/data.py
 e396b7971d38896e0e20b973a3a6a3fbc3171d080a21bc6e66a65bee452fd69c  lib/core/datatype.py
 e18c0c2c5a57924a623792a48bfd36e98d9bc085f6db61a95fc0dc8a3bcedc0c  lib/core/decorators.py
 147823c37596bd6a56d677697781f34b8d1d1671d5a2518fbc9468d623c6d07d  lib/core/defaults.py
 76756ce056dd2e5363e5e1d2b2b4c3f3b36ff8c8ac215e83ee3b8d535ea6b741  lib/core/dicts.py
-143f0b5b30e0f24b3527945879497de1542ba1aee9e6f0f98865e626c4c9317e  lib/core/dump.py
+a033f92d136c707a25927c2383125ddb004d4283db62c004dcd67c3fc242bb1c  lib/core/dump.py
 1abf1edeacb85eaf5cffd35fcbde4eee2da6f5fc722a8dc1f9287fb55d138418  lib/core/enums.py
 5387168e5dfedd94ae22af7bb255f27d6baaca50b24179c6b98f4f325f5cc7b4  lib/core/exception.py
 1966ca704961fb987ab757f0a4afddbf841d1a880631b701487c75cef63d60c3  lib/core/__init__.py
 914a13ee21fd610a6153a37cbe50830fcbd1324c7ebc1e7fc206d5e598b0f7ad  lib/core/log.py
 02a2264324caa249154e024a01bcd7cc40dbca4d647d5d10a50654b4415a6d77  lib/core/optiondict.py
-6576d40a66fa7871d3498c193f4e1e50a9fa9a380005d019c5c2266c1dc31c21  lib/core/option.py
+4dc6383b5d9cfff5b1bf8ffa8b0b02a7dbd8df8624b76030f8925c2dbdee9bf9  lib/core/option.py
 8171f6ee33e7742f06bb3014a28324496374beddee7b378ace10a26414a97762  lib/core/patch.py
 49c0fa7e3814dfda610d665ee02b12df299b28bc0b6773815b4395514ddf8dec  lib/core/profiling.py
 03db48f02c3d07a047ddb8fe33a757b6238867352d8ddda2a83e4fec09a98d04  lib/core/readlineng.py
 48797d6c34dd9bb8a53f7f3794c85f4288d82a9a1d6be7fcf317d388cb20d4b3  lib/core/replication.py
 3574639db4942d16a2dc0a2f04bb7c0913c40c3862b54d34c44075a760e0c194  lib/core/revision.py
 888daba83fd4a34e9503fe21f01fef4cc730e5cde871b1d40e15d4cbc847d56c  lib/core/session.py
-9d05d48a24cf166f965ae95e63a57e92061bc84bb1fc2b3fe1c5ee985e17b453  lib/core/settings.py
+2503c9b9eca74ae3c2a6b245504046b623bf0e8a8f433105f41f61fa626aec48  lib/core/settings.py
 cd5a66deee8963ba8e7e9af3dd36eb5e8127d4d68698811c29e789655f507f82  lib/core/shell.py
 bcb5d8090d5e3e0ef2a586ba09ba80eef0c6d51feb0f611ed25299fbb254f725  lib/core/subprocessng.py
 d35650179816193164a5f177102f18379dfbe6bb6d40fbb67b78d907b41c8038  lib/core/target.py
@@ -201,7 +201,7 @@ ec11fd5a3f4efd10a1cae288157ac6eb6fb75da4666d76d19f6adf74ac338b5a  lib/core/wordl
 1966ca704961fb987ab757f0a4afddbf841d1a880631b701487c75cef63d60c3  lib/__init__.py
 54bfd31ebded3ffa5848df1c644f196eb704116517c7a3d860b5d081e984d821  lib/parse/banner.py
 a9f10a558684778bdb00d446cb88967fc1bfd413ae6a5f4bd582b3ea442baa87  lib/parse/cmdline.py
-cac08047db1168b24174496268af811cee555e7c3fdd528ef8d895ec49b91d36  lib/parse/configfile.py
+02d82e4069bd98c52755417f8b8e306d79945672656ac24f1a45e7a6eff4b158  lib/parse/configfile.py
 c5b258be7485089fac9d9cd179960e774fbd85e62836dc67cce76cc028bb6aeb  lib/parse/handler.py
 97361d481a97b600a3086b7f228f54ffa68a78df8b63b76bfaa5495d66770b63  lib/parse/headers.py
 1ad9054cd8476a520d4e2c141085ae45d94519df5c66f25fac41fe7d552ab952  lib/parse/html.py
@@ -212,7 +212,7 @@ d2e771cdacef25ee3fdc0e0355b92e7cd1b68f5edc2756ffc19f75d183ba2c73  lib/parse/payl
 a1c638493ecdc5194db7186bbfed815c6eed2344f2607cac8c9fa50534824266  lib/request/basic.py
 bc61bc944b81a7670884f82231033a6ac703324b34b071c9834886a92e249d0e  lib/request/chunkedhandler.py
 2daf0ce19eacda64687f441c90ef8da51714c3e8947c993ba08fb4ecdc4f5287  lib/request/comparison.py
-626bb6f3316a906a4629c0feb8ecbbcf473fb59e5bc532603c35b6b8f63f1deb  lib/request/connect.py
+f83140c85be7f572f83c4ab4279fa1d8601243210cdfe4a44b2fc218befbcffd  lib/request/connect.py
 8e06682280fce062eef6174351bfebcb6040e19976acff9dc7b3699779783498  lib/request/direct.py
 cf019248253a5d7edb7bc474aa020b9e8625d73008a463c56ba2b539d7f2d8ec  lib/request/dns.py
 f56fc33251bd6214e3a6316c8f843eb192b2996aa84bd4c3e98790fdcf6e8cf0  lib/request/httpshandler.py
@@ -244,7 +244,7 @@ f552b6140d4069be6a44792a08f295da8adabc1c4bb6a5e100f222f87144ca9d  lib/techniques
 a17c1d201bd084de0093254bcd303aa859399891de13a7259e8c200e98294efb  lib/techniques/union/use.py
 67dff80a17503b91c8ff93788ccc037b6695aa18b0793894b42488cbb21c4c83  lib/utils/api.py
 ea5e14f8c9d74b0fb17026b14e3fb70ee90e4046e51ab2c16652d86b3ca9b949  lib/utils/brute.py
-3fa1b9fd57ff47c6a283e8381bf70259dce57bb2327f99d8cb56450f1acf2d46  lib/utils/crawler.py
+da5bcbcda3f667582adf5db8c1b5d511b469ac61b55d387cec66de35720ed718  lib/utils/crawler.py
 a94958be0ec3e9d28d8171813a6a90655a9ad7e6aa33c661e8d8ebbfcf208dbb  lib/utils/deps.py
 51cfab194cd5b6b24d62706fb79db86c852b9e593f4c55c15b35f175e70c9d75  lib/utils/getch.py
 853c3595e1d2efc54b8bfb6ab12c55d1efc1603be266978e3a7d96d553d91a52  lib/utils/gui.py
@@ -255,7 +255,7 @@ dc68b7fdb2ae0e958f2a553984a94a06832d216a3209f632ad9ff53f17554524  lib/utils/http
 1966ca704961fb987ab757f0a4afddbf841d1a880631b701487c75cef63d60c3  lib/utils/__init__.py
 22ba65391b0a73b1925e5becf8ddab6ba73a196d86e351a2263509aad6676bd7  lib/utils/pivotdumptable.py
 c1dfc3bed0fed9b181f612d1d747955dd2b506dbe99bc9fd481495602371473a  lib/utils/progress.py
-b0f777739420f7207e57d7073fd94c630a456e4c419d440b98ed6521143952dc  lib/utils/purge.py
+27afe211030d06db28df85296bfbf698296c94440904c390cef0ff0c259dbbc5  lib/utils/purge.py
 c853aa08ab24a00a78969408d60684da0ccb33a2a6693492e0acb7c480ffbcd1  lib/utils/safe2bin.py
 2ee72e83500a1bf02fcd942564fca0053a0c46f736286f0c35dd6904e09f4734  lib/utils/search.py
 8258d0f54ad94e6101934971af4e55d5540f217c40ddcc594e2fba837b856d35  lib/utils/sgmllib.py
@@ -487,7 +487,7 @@ eb45fd711efa71ab9d91d815cc8abebc9abc4770311fbb827159008b000f4fc2  plugins/generi
 c6ad39bfd1810413402dedfc275fc805fa13f85fc490e236c1e725bde4e5100b  sqlmapapi.py
 4e993cfe2889bf0f86ad0abafd9a6a25849580284ea279b2115e99707e14bb97  sqlmapapi.yaml
 a40607ce164eb2d21865288d24b863edb1c734b56db857e130ac1aef961c80b9  sqlmap.conf
-e9d3d52d4c0698b956cc0dc92c177d432b1f97c5918f750baa3e737de4ae574b  sqlmap.py
+01457a41833b89b8bcdcae7b241a184bfcfdd8088575506154dd6ea1f4d71186  sqlmap.py
 eb37a88357522fd7ad00d90cdc5da6b57442b4fec49366aadb2944c4fbf8b804  tamper/0eunion.py
 a9785a4c111d6fee2e6d26466ba5efb3b229c00520b26e8024b041553b53efba  tamper/apostrophemask.py
 cf26bc8006519bd25ce06d347f72770cd75b61575cf65e5812274e8ab9392eb4  tamper/apostrophenullencode.py

lib/core/common.py

@@ -2465,7 +2465,7 @@ def getSQLSnippet(dbms, sfile, **variables):
 
     return retVal
 
-def readCachedFileContent(filename, mode="rb"):
+def readCachedFileContent(filename, mode='r'):
     """
     Cached reading of file content (avoiding multiple same file reading)
 
@@ -3609,7 +3609,7 @@ def saveConfig(conf, filename):
 
             config.set(family, option, value)
 
-    with openFile(filename, "wb") as f:
+    with openFile(filename, 'w') as f:
         try:
             config.write(f)
         except IOError as ex:
@@ -3815,6 +3815,7 @@ def openFile(filename, mode='r', encoding=UNICODE_ENCODING, errors="reversible",
     # Reference: https://stackoverflow.com/a/37462452
     if 'b' in mode:
         buffering = 0
+        encoding = None
 
     if filename == STDIN_PIPE_DASH:
         if filename not in kb.cache.content:
@@ -4022,7 +4023,7 @@ def createGithubIssue(errMsg, excMsg):
             logger.info(infoMsg)
 
             try:
-                with openFile(paths.GITHUB_HISTORY, "a+b") as f:
+                with openFile(paths.GITHUB_HISTORY, "a+") as f:
                     f.write("%s\n" % key)
             except:
                 pass
@@ -5103,7 +5104,7 @@ def resetCookieJar(cookieJar):
                 os.close(handle)
 
                 # Reference: http://www.hashbangcode.com/blog/netscape-http-cooke-file-parser-php-584.html
-                with openFile(filename, "w+b") as f:
+                with openFile(filename, "w+") as f:
                     f.write("%s\n" % NETSCAPE_FORMAT_HEADER_COOKIES)
                     for line in lines:
                         _ = line.split("\t")

lib/core/dump.py

@@ -110,7 +110,7 @@ def setOutputFile(self):
 
         self._outputFile = os.path.join(conf.outputPath, "log")
         try:
-            self._outputFP = openFile(self._outputFile, "ab" if not conf.flushSession else "wb")
+            self._outputFP = openFile(self._outputFile, 'a' if not conf.flushSession else 'w')
         except IOError as ex:
             errMsg = "error occurred while opening log file ('%s')" % getSafeExString(ex)
             raise SqlmapGenericException(errMsg)
@@ -453,7 +453,7 @@ def dbTableValues(self, tableValues):
             dumpFileName = conf.dumpFile or os.path.join(dumpDbPath, re.sub(r'[\\/]', UNSAFE_DUMP_FILEPATH_REPLACEMENT, "%s.%s" % (unsafeSQLIdentificatorNaming(table), conf.dumpFormat.lower())))
             if not checkFile(dumpFileName, False):
                 try:
-                    openFile(dumpFileName, "w+b").close()
+                    openFile(dumpFileName, "w+").close()
                 except SqlmapSystemException:
                     raise
                 except:
@@ -481,7 +481,7 @@ def dbTableValues(self, tableValues):
                         else:
                             count += 1
 
-            dumpFP = openFile(dumpFileName, "wb" if not appendToFile else "ab", buffering=DUMP_FILE_BUFFER_SIZE)
+            dumpFP = openFile(dumpFileName, 'w' if not appendToFile else 'a', buffering=DUMP_FILE_BUFFER_SIZE)
 
         count = int(tableValues["__infos__"]["count"])
         if count > TRIM_STDOUT_DUMP_SIZE:

lib/core/option.py

@@ -753,7 +753,7 @@ def _listTamperingFunctions():
         logger.info(infoMsg)
 
         for script in sorted(glob.glob(os.path.join(paths.SQLMAP_TAMPER_PATH, "*.py"))):
-            content = openFile(script, "rb").read()
+            content = openFile(script, 'r').read()
             match = re.search(r'(?s)__priority__.+"""(.+)"""', content)
             if match:
                 comment = match.group(1).strip()
@@ -1015,8 +1015,8 @@ def _setPostprocessFunctions():
                     handle, filename = tempfile.mkstemp(prefix=MKSTEMP_PREFIX.PREPROCESS, suffix=".py")
                     os.close(handle)
 
-                    openFile(filename, "w+b").write("#!/usr/bin/env\n\ndef postprocess(page, headers=None, code=None):\n    return page, headers, code\n")
-                    openFile(os.path.join(os.path.dirname(filename), "__init__.py"), "w+b").write("pass")
+                    openFile(filename, "w+").write("#!/usr/bin/env\n\ndef postprocess(page, headers=None, code=None):\n    return page, headers, code\n")
+                    openFile(os.path.join(os.path.dirname(filename), "__init__.py"), "w+").write("pass")
 
                     errMsg = "function 'postprocess(page, headers=None, code=None)' "
                     errMsg += "in postprocess script '%s' " % script
@@ -1593,7 +1593,7 @@ def _createHomeDirectories():
                 os.makedirs(directory)
 
             _ = os.path.join(directory, randomStr())
-            open(_, "w+b").close()
+            open(_, "w+").close()
             os.remove(_)
 
             if conf.get("outputDir") and context == "output":
@@ -1623,7 +1623,7 @@ def _createTemporaryDirectory():
 
             _ = os.path.join(conf.tmpDir, randomStr())
 
-            open(_, "w+b").close()
+            open(_, "w+").close()
             os.remove(_)
 
             tempfile.tempdir = conf.tmpDir

lib/core/settings.py

@@ -19,7 +19,7 @@
 from thirdparty import six
 
 # sqlmap version (<major>.<minor>.<month>.<monthly commit>)
-VERSION = "1.10.1.51"
+VERSION = "1.10.1.52"
 TYPE = "dev" if VERSION.count('.') > 2 and VERSION.split('.')[-1] != '0' else "stable"
 TYPE_COLORS = {"dev": 33, "stable": 90, "pip": 34}
 VERSION_STRING = "sqlmap/%s#%s" % ('.'.join(VERSION.split('.')[:-1]) if VERSION.count('.') > 2 and VERSION.split('.')[-1] == '0' else VERSION, TYPE)

lib/parse/configfile.py

@@ -64,7 +64,7 @@ def configFileParser(configFile):
     logger.debug(debugMsg)
 
     checkFile(configFile)
-    configFP = openFile(configFile, "rb")
+    configFP = openFile(configFile, 'r')
 
     try:
         config = UnicodeRawConfigParser()

lib/request/connect.py

@@ -490,7 +490,7 @@ def getPage(**kwargs):
                 headers = forgeHeaders(auxHeaders, headers)
 
             if kb.headersFile:
-                content = openFile(kb.headersFile, "rb").read()
+                content = openFile(kb.headersFile, 'r').read()
                 for line in content.split("\n"):
                     line = getText(line.strip())
                     if ':' in line:

lib/utils/crawler.py

@@ -254,7 +254,7 @@ def storeResultsToFile(results):
         infoMsg = "writing crawling results to a temporary file '%s' " % filename
         logger.info(infoMsg)
 
-        with openFile(filename, "w+b") as f:
+        with openFile(filename, "w+") as f:
             if conf.forms:
                 f.write("URL,POST\n")
 

lib/utils/purge.py

@@ -50,7 +50,7 @@ def purge(directory):
     for filepath in filepaths:
         try:
             filesize = os.path.getsize(filepath)
-            with openFile(filepath, "w+b") as f:
+            with openFile(filepath, "w+") as f:
                 f.write("".join(_unichr(random.randint(0, 255)) for _ in xrange(filesize)))
         except:
             pass

sqlmap.py

@@ -593,7 +593,7 @@ def main():
 
         if conf.get("harFile"):
             try:
-                with openFile(conf.harFile, "w+b") as f:
+                with openFile(conf.harFile, "w+") as f:
                     json.dump(conf.httpCollector.obtain(), fp=f, indent=4, separators=(',', ': '))
             except SqlmapBaseException as ex:
                 errMsg = getSafeExString(ex)