Differences

This shows you the differences between two versions of the page.

--- python:internet [2014/08/06 03:56] – [urllib2.build_opener(replace urlopen) and post with cookies] admin
+++ python:internet [2022/10/29 16:15] (current) – external edit 127.0.0.1
@@ Line 131: / Line 131: @@
 </BODY></HTML>
 </code>
-===== httplib and urllib2 =====
+===== httplib and urllib2(with addinfourl) =====
   * The basic class for processing http is httplib package
   * urllib2 is a Python module that can be used for fetching URLs. It defines functions and classes to help with URL actions (basic and digestauthentication, redirections, cookies, etc). urllib2 handle object types below: <code python>
@@ Line 252: / Line 252: @@
 response = opener.open('http://www.python.org')
 print response.read()
+</code> and for sending post:<code python>
+import urllib, urllib2
+params = urllib.urlencode({'@number': 12524, '@type': 'issue', '@action': 'show'})
+headers = {"Content-type": "application/x-www-form-urlencoded",
+           "Accept": "text/plain"}
+req = urllib2.Request("http://bugs.python.org", data = params, headers = headers)
+opener = urllib2.build_opener(urllib2.HTTPHandler())
+response = opener.open(req)
+print 'headers:', response.info()
+print response.read()
+</code> output:<code>
+headers: Date: Wed, 06 Aug 2014 04:01:39 GMT
+Server: BaseHTTP/0.3 Python/2.6.6
+Expires: Wed, 06 Aug 2014 04:01:38 GMT
+Content-Type: text/html; charset=utf-8
+Cache-Control: no-cache
+Via: 1.1 bugs.python.org
+Connection: close
+Transfer-Encoding: chunked
 </code>
-  * Send login to OpenCart:<code python>
+  * Send normal login to OpenCart:<code python>
 import urllib, urllib2
@@ Line 260: / Line 280: @@
 headers = {"Content-type": "application/x-www-form-urlencoded",
            "Accept": "text/plain"}
 req = urllib2.Request(loginurl, data = params, headers = headers)
 response = urllib2.urlopen(req)
-print 'headers:', response.info()
 data = response.read()
 with open('loginresult.html', 'wb') as f:
     f.write(data)
+</code>Debug in httplib.HTTPConnection send:<code>
+POST /index.php?route=account/login HTTP/1.1
+Accept-Encoding: identity
+Content-Length: 122
+Host: dacsan.babies.vn
+Accept: text/plain
+User-Agent: Python-urllib/2.7
+Connection: close
+Content-Type: application/x-www-form-urlencoded
+redirect=http%3A%2F%2Fdacsan.babies.vn%2Findex.php%3Froute%3Daccount%2Faccount&password=8941362&email=itanhchi%40yahoo.com
+GET /index.php?route=account/account HTTP/1.1
+Accept-Encoding: identity
+Host: dacsan.babies.vn
+Connection: close
+Accept: text/plain
+User-Agent: Python-urllib/2.7
+GET /index.php?route=account/login HTTP/1.1
+Accept-Encoding: identity
+Host: dacsan.babies.vn
+Connection: close
+Accept: text/plain
+User-Agent: Python-urllib/2.7
 </code>
+  * Send login to OpenCart with cookies header:<code python>
+import urllib, urllib2
+loginurl = 'http://dacsan.babies.vn/index.php?route=account/login'
+params = urllib.urlencode({'email': 'itanhchi@yahoo.com', 'password':'8941362', 'redirect':'http://dacsan.babies.vn/index.php?route=account/account'})
+headers = {"Content-type": "application/x-www-form-urlencoded",
+           "Accept": "text/plain"}
+req = urllib2.Request(loginurl, data = params, headers = headers)
+opener = urllib2.build_opener(urllib2.HTTPCookieProcessor())
+response = opener.open(req)
+data = response.read()
+with open('loginresult.html', 'wb') as f:
+    f.write(data)
+</code>Debug in httplib.HTTPConnection send:<code>
+POST /index.php?route=account/login HTTP/1.1
+Accept-Encoding: identity
+Content-Length: 122
+Host: dacsan.babies.vn
+Accept: text/plain
+User-Agent: Python-urllib/2.7
+Connection: close
+Content-Type: application/x-www-form-urlencoded
+redirect=http%3A%2F%2Fdacsan.babies.vn%2Findex.php%3Froute%3Daccount%2Faccount&password=8941362&email=itanhchi%40yahoo.com
+GET /index.php?route=account/account HTTP/1.1
+Accept-Encoding: identity
+Host: dacsan.babies.vn
+Cookie: currency=VND; language=vn; PHPSESSID=h1mr19s902mekq25ubccsn4k22
+Connection: close
+Accept: text/plain
+User-Agent: Python-urllib/2.7
+</code>
+==== Get Cookie Information ====
+<code python>
+import urllib, urllib2
+import cookielib
+loginurl = 'http://dacsan.babies.vn/index.php?route=account/login'
+params = urllib.urlencode({'email': 'itanhchi@yahoo.com', 'password':'8941362', 'redirect':'http://dacsan.babies.vn/index.php?route=account/account'})
+headers = {"Content-type": "application/x-www-form-urlencoded",
+           "Accept": "text/plain"}
+req = urllib2.Request(loginurl, data = params, headers = headers)
+cj = cookielib.CookieJar()
+opener = urllib2.build_opener(urllib2.HTTPCookieProcessor(cj))
+response = opener.open(req)
+for cookie in cj:
+    print cookie.name, cookie.value
+data = response.read()
+with open('loginresult.html', 'wb') as f:
+    f.write(data)
+</code>output:<code>
+currency VND
+language vn
+PHPSESSID 01eg0u7uf5bm3r01h6pnrv3q33
+</code>
+==== Post multipart form data ====
+=== post with encode_multipart_formdata ===
+Post with httplib<code python>
+import httplib, mimetypes
+def post_multipart(host, selector, fields, files):
+    """
+    Post fields and files to an http host as multipart/form-data.
+    fields is a sequence of (name, value) elements for regular form fields.
+    files is a sequence of (name, filename, value) elements for data to be uploaded as files
+    Return the server's response page.
+    """
+    content_type, body = encode_multipart_formdata(fields, files)
+    h = httplib.HTTP(host)
+    h.putrequest('POST', selector)
+    h.putheader('content-type', content_type)
+    h.putheader('content-length', str(len(body)))
+    h.endheaders()
+    h.send(body)
+    errcode, errmsg, headers = h.getreply()
+    return h.file.read()
+def encode_multipart_formdata(fields, files):
+    """
+    fields is a sequence of (name, value) elements for regular form fields.
+    files is a sequence of (name, filename, value) elements for data to be uploaded as files
+    Return (content_type, body) ready for httplib.HTTP instance
+    """
+    BOUNDARY = '----------ThIs_Is_tHe_bouNdaRY_$'
+    CRLF = '\r\n'
+    L = []
+    for (key, value) in fields:
+        L.append('--' + BOUNDARY)
+        L.append('Content-Disposition: form-data; name="%s"' % key)
+        L.append('')
+        L.append(value)
+    for (key, filename, value) in files:
+        L.append('--' + BOUNDARY)
+        L.append('Content-Disposition: form-data; name="%s"; filename="%s"' % (key, filename))
+        L.append('Content-Type: %s' % get_content_type(filename))
+        L.append('')
+        L.append(value)
+    L.append('--' + BOUNDARY + '--')
+    L.append('')
+    body = CRLF.join(L)
+    content_type = 'multipart/form-data; boundary=%s' % BOUNDARY
+    return content_type, body
+def get_content_type(filename):
+    return mimetypes.guess_type(filename)[0] or 'application/octet-stream'
+</code>
+=== fix encode_multipart_formdata for posting binary file ===
+<code python>
+def encode_multipart_formdata(fields, files):
+    """
+    fields is a sequence of (name, value) elements for regular form fields.
+    files is a sequence of (name, filename, value) elements for data to be uploaded as files
+    Return (content_type, body) ready for httplib.HTTP instance
+    """
+    buf = StringIO()
+    boundary = mimetools.choose_boundary()
+    for (key, value) in fields:
+        buf.write('--%s\r\n' % boundary)
+        buf.write('Content-Disposition: form-data; name="%s"' % key)
+        buf.write('\r\n\r\n' + value + '\r\n')
+    for (key, filename, value) in files:
+        contenttype = mimetypes.guess_type(filename)[0] or 'application/octet-stream'
+        buf.write('--%s\r\n' % boundary)
+        buf.write('Content-Disposition: form-data; name="%s"; filename="%s"\r\n' % (key, filename))
+        buf.write('Content-Type: %s\r\n' % contenttype)
+        buf.write('\r\n' + value + '\r\n')
+    buf.write('--' + boundary + '--\r\n\r\n')
+    buf = buf.getvalue()
+    content_type = 'multipart/form-data; boundary=%s' % boundary
+    return content_type, buf
+</code>
+=== Post using MultipartPostHandler ===
+  * Install:<code bash>
+pip install MultipartPostHandler
+</code>
+  * example:<code python>
+import MultipartPostHandler, urllib2, cookielib
+cookies = cookielib.CookieJar()
+opener = urllib2.build_opener(urllib2.HTTPCookieProcessor(cookies),
+                                MultipartPostHandler.MultipartPostHandler)
+params = { "username" : "bob", "password" : "riviera",
+           "file" : open("filename", "rb") }
+opener.open("http://wwww.bobsite.com/upload/", params)
+</code>
+===== twisted internet =====
+==== Simple Request ====
+<code python>
+from sys import argv
+from pprint import pformat
+from twisted.internet.task import react
+from twisted.web.client import Agent, readBody
+from twisted.web.http_headers import Headers
+def cbRequest(response):
+    print 'Response version:', response.version
+    print 'Response code:', response.code
+    print 'Response phrase:', response.phrase
+    print 'Response headers:'
+    print pformat(list(response.headers.getAllRawHeaders()))
+    d = readBody(response)
+    d.addCallback(cbBody)
+    return d
+def cbBody(body):
+    print 'Response body:'
+    print body
+def main(reactor, url=b"http://www.google.com/"):
+    agent = Agent(reactor)
+    d = agent.request(
+        'GET', url,
+        Headers({'User-Agent': ['Twisted Web Client Example']}),
+        None)
+    d.addCallback(cbRequest)
+    return d
+react(main, argv[1:])
+</code>