Improving keep-alive support

Author: stamparm

data/txt/sha256sums.txt

@@ -188,7 +188,7 @@ c1cb56f2a43e9f2f6b25d5f3d504e856ea21df6fc14af5e37b1000feef2bdb5a  lib/core/optio
 48797d6c34dd9bb8a53f7f3794c85f4288d82a9a1d6be7fcf317d388cb20d4b3  lib/core/replication.py
 0b8c38a01bb01f843d94a6c5f2075ee47520d0c4aa799cecea9c3e2c5a4a23a6  lib/core/revision.py
 888daba83fd4a34e9503fe21f01fef4cc730e5cde871b1d40e15d4cbc847d56c  lib/core/session.py
-1418691b5449412e60c693b6afc2f12b00051c1e280d2261762a36f094e0da66  lib/core/settings.py
+bf818add365e18e378b15fb33db123d846acddc2969e05af52eacfe745cc335e  lib/core/settings.py
 cd5a66deee8963ba8e7e9af3dd36eb5e8127d4d68698811c29e789655f507f82  lib/core/shell.py
 bcb5d8090d5e3e0ef2a586ba09ba80eef0c6d51feb0f611ed25299fbb254f725  lib/core/subprocessng.py
 d35650179816193164a5f177102f18379dfbe6bb6d40fbb67b78d907b41c8038  lib/core/target.py
@@ -618,7 +618,7 @@ edf23e7105539d700a1ae1bc52436e57e019b345a7d0227e4d85b6353ef535fa  thirdparty/ide
 d846fdc47a11a58da9e463a948200f69265181f3dbc38148bfe4141fade10347  thirdparty/identywaf/LICENSE
 e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855  thirdparty/__init__.py
 879d96f2460bc6c79c0db46b5813080841c7403399292ce76fe1dc0a6ed353d8  thirdparty/keepalive/__init__.py
-f517561115b0cfaa509d0d4216cd91c7de92c6a5a30f1688fdca22e4cd52b8f8  thirdparty/keepalive/keepalive.py
+c7ac7253fa450030f9c42f11bb19689055bb8c39621bcfbeca856ba3c9342760  thirdparty/keepalive/keepalive.py
 e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855  thirdparty/magic/__init__.py
 4d89a52f809c28ce1dc17bb0c00c775475b8ce01c2165942877596a6180a2fd8  thirdparty/magic/magic.py
 e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855  thirdparty/multipart/__init__.py

lib/core/settings.py

@@ -20,7 +20,7 @@
 from thirdparty import six
 
 # sqlmap version (<major>.<minor>.<month>.<monthly commit>)
-VERSION = "1.10.1.85"
+VERSION = "1.10.2.0"
 TYPE = "dev" if VERSION.count('.') > 2 and VERSION.split('.')[-1] != '0' else "stable"
 TYPE_COLORS = {"dev": 33, "stable": 90, "pip": 34}
 VERSION_STRING = "sqlmap/%s#%s" % ('.'.join(VERSION.split('.')[:-1]) if VERSION.count('.') > 2 and VERSION.split('.')[-1] == '0' else VERSION, TYPE)

thirdparty/keepalive/keepalive.py

@@ -12,9 +12,9 @@
 #   Lesser General Public License for more details.
 #
 #   You should have received a copy of the GNU Lesser General Public
-#   License along with this library; if not, write to the 
-#      Free Software Foundation, Inc., 
-#      59 Temple Place, Suite 330, 
+#   License along with this library; if not, write to the
+#      Free Software Foundation, Inc.,
+#      59 Temple Place, Suite 330,
 #      Boston, MA  02111-1307  USA
 
 # This file was part of urlgrabber, a high-level cross-protocol url-grabber
@@ -28,7 +28,7 @@
 >>> keepalive_handler = HTTPHandler()
 >>> opener = _urllib.request.build_opener(keepalive_handler)
 >>> _urllib.request.install_opener(opener)
->>> 
+>>>
 >>> fo = _urllib.request.urlopen('http://www.python.org')
 
 If a connection to a given host is requested, and all of the existing
@@ -154,14 +154,18 @@ def remove(self, connection):
             else:
                 del self._connmap[connection]
                 del self._readymap[connection]
-                self._hostmap[host].remove(connection)
+                try:
+                    self._hostmap[host].remove(connection)
+                except ValueError:
+                    pass
                 if not self._hostmap[host]: del self._hostmap[host]
         finally:
             self._lock.release()
 
     def set_ready(self, connection, ready):
-        try: self._readymap[connection] = ready
-        except KeyError: pass
+        self._lock.acquire()
+        if connection in self._readymap: self._readymap[connection] = ready
+        self._lock.release()
 
     def get_ready_conn(self, host):
         conn = None
@@ -178,10 +182,14 @@ def get_ready_conn(self, host):
         return conn
 
     def get_all(self, host=None):
-        if host:
-            return list(self._hostmap.get(host, []))
-        else:
-            return dict(self._hostmap)
+        self._lock.acquire()
+        try:
+            if host:
+                return list(self._hostmap.get(host, []))
+            else:
+                return dict(self._hostmap)
+        finally:
+            self._lock.release()
 
 class KeepAliveHandler:
     def __init__(self):
@@ -242,9 +250,9 @@ def do_open(self, req):
                 h = self._get_connection(host)
                 if DEBUG: DEBUG.info("creating new connection to %s (%d)",
                                      host, id(h))
-                self._cm.add(host, h, 0)
                 self._start_transaction(h, req)
                 r = h.getresponse()
+                self._cm.add(host, h, 0)
         except (socket.error, _http_client.HTTPException) as err:
             raise _urllib.error.URLError(err)
 
@@ -254,20 +262,20 @@ def do_open(self, req):
         if r.will_close:
             if DEBUG: DEBUG.info('server will close connection, discarding')
             self._cm.remove(h)
+            h.close()
 
         r._handler = self
         r._host = host
         r._url = req.get_full_url()
         r._connection = h
         r.code = r.status
         r.headers = r.msg
-        r.msg = r.reason
 
         if r.status == 200 or not HANDLE_ERRORS:
             return r
         else:
             return self.parent.error('http', req, r,
-                                     r.status, r.msg, r.headers)
+                                     r.status, r.reason, r.headers)
 
     def _reuse_connection(self, h, req, host):
         """start the transaction with a re-used connection
@@ -283,7 +291,7 @@ def _reuse_connection(self, h, req, host):
             # worked.  We'll check the version below, too.
         except (socket.error, _http_client.HTTPException):
             r = None
-        except:
+        except Exception:
             # adding this block just in case we've missed
             # something we will still raise the exception, but
             # lets try and close the connection and remove it
@@ -314,16 +322,16 @@ def _reuse_connection(self, h, req, host):
 
     def _start_transaction(self, h, req):
         try:
-            if req.data:
+            if req.data is not None:
                 data = req.data
                 if hasattr(req, 'selector'):
                     h.putrequest(req.get_method() or 'POST', req.selector, skip_host=req.has_header("Host"), skip_accept_encoding=req.has_header("Accept-encoding"))
                 else:
                     h.putrequest(req.get_method() or 'POST', req.get_selector(), skip_host=req.has_header("Host"), skip_accept_encoding=req.has_header("Accept-encoding"))
-                if 'Content-type' not in req.headers:
+                if not req.has_header('Content-type'):
                     h.putheader('Content-type',
                                 'application/x-www-form-urlencoded')
-                if 'Content-length' not in req.headers:
+                if not req.has_header('Content-length'):
                     h.putheader('Content-length', '%d' % len(data))
             else:
                 if hasattr(req, 'selector'):
@@ -333,20 +341,20 @@ def _start_transaction(self, h, req):
         except (socket.error, _http_client.HTTPException) as err:
             raise _urllib.error.URLError(err)
 
-        if 'Connection' not in req.headers:
-            req.headers['Connection'] = 'keep-alive'
+        if not req.has_header('Connection'):
+            h.putheader('Connection', 'keep-alive')
 
         for args in self.parent.addheaders:
-            if args[0] not in req.headers:
+            if not req.has_header(args[0]):
                 h.putheader(*args)
         for k, v in req.headers.items():
             h.putheader(k, v)
         h.endheaders()
-        if req.data:
+        if req.data is not None:
             h.send(data)
 
     def _get_connection(self, host):
-        return NotImplementedError
+        raise NotImplementedError()
 
 class HTTPHandler(KeepAliveHandler, _urllib.request.HTTPHandler):
     def __init__(self):
@@ -373,8 +381,10 @@ def https_open(self, req):
         return self.do_open(req)
 
     def _get_connection(self, host):
-        try: return self._ssl_factory.get_https_connection(host)
-        except AttributeError: return HTTPSConnection(host)
+        if self._ssl_factory:
+            return self._ssl_factory.get_https_connection(host)
+        else:
+            return HTTPSConnection(host)
 
 class HTTPResponse(_http_client.HTTPResponse):
     # we need to subclass HTTPResponse in order to
@@ -397,9 +407,9 @@ class HTTPResponse(_http_client.HTTPResponse):
 
 
     def __init__(self, sock, debuglevel=0, strict=0, method=None):
-        if method: # the httplib in python 2.3 uses the method arg
-            _http_client.HTTPResponse.__init__(self, sock, debuglevel, method)
-        else: # 2.2 doesn't
+        if method:
+            _http_client.HTTPResponse.__init__(self, sock, debuglevel, method=method)
+        else:
             _http_client.HTTPResponse.__init__(self, sock, debuglevel)
         self.fileno = sock.fileno
         self.code = None
@@ -453,11 +463,11 @@ def read(self, amt=None):
 
     def readline(self, limit=-1):
         data = b""
-        i = self._rbuf.find('\n')
+        i = self._rbuf.find(b'\n')
         while i < 0 and not (0 < limit <= len(self._rbuf)):
             new = self._raw_read(self._rbufsize)
             if not new: break
-            i = new.find('\n')
+            i = new.find(b'\n')
             if i >= 0: i = i + len(self._rbuf)
             self._rbuf = self._rbuf + new
         if i < 0: i = len(self._rbuf)
@@ -468,15 +478,15 @@ def readline(self, limit=-1):
 
     def readlines(self, sizehint = 0):
         total = 0
-        list = []
+        lines = []
         while 1:
             line = self.readline()
             if not line: break
-            list.append(line)
+            lines.append(line)
             total += len(line)
             if sizehint and total >= sizehint:
                 break
-        return list
+        return lines
 
 
 class HTTPConnection(_http_client.HTTPConnection):
@@ -540,10 +550,10 @@ def continuity(url):
     print(format % ('keepalive read', m.hexdigest()))
 
     fo = _urllib.request.urlopen(url)
-    foo = ''
+    foo = b''
     while 1:
         f = fo.readline()
-        if f: foo = foo + f
+        if f: foo += f
         else: break
     fo.close()
     m = md5(foo)