my notes

User Tools

Site Tools

Trace: • internet
python:internet

Table of Contents

Python Internet

Python Socket

refer: https://docs.python.org/2/library/socket.html

Echo Server and Client

Simple Echo Server and Client

  • Echo Server
    # Echo server program
import socket
 
HOST = ''                 # Symbolic name meaning all available interfaces
PORT = 50007              # Arbitrary non-privileged port
s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
s.bind((HOST, PORT))
s.listen(1)
print 'server is listening'
conn, addr = s.accept()
print 'Connected by', addr
while 1:
    data = conn.recv(1024)
    if not data: break
    conn.sendall(data)
conn.close()

    output: 

    server is listening
  • Echo Client
    # Echo client program
import socket
 
HOST = '127.0.0.1'    # The remote host
PORT = 50007              # The same port as used by the server
s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
s.connect((HOST, PORT))
s.sendall('Hello, world')
data = s.recv(1024)
s.close()
print 'Received', repr(data)

    server output: 

    server is listening
Connected by ('127.0.0.1', 56354)

    client output: 

    Received 'Hello, world'

Custom Echo Server for multiple client

The above example only service for 1 client, after the server receive connection and send data to client, the echoserver program will end. The below example will service for multiple clients

  • Echo server
    # Echo server program
import socket
 
HOST = ''                 # Symbolic name meaning all available interfaces
PORT = 50007              # Arbitrary non-privileged port
s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
s.bind((HOST, PORT))
s.listen(10)
print 'server is listening'
while 1:
    conn, addr = s.accept()
    print 'Connected by', addr
    data = conn.recv(1024)
    if not data: break
    conn.sendall(data)
conn.close()

Connect to google with socket and receive data

#Socket client example in python
 
import socket   #for sockets
import sys  #for exit
 
#create an INET, STREAMing socket
try:
    s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
except socket.error:
    print 'Failed to create socket'
    sys.exit()
 
print 'Socket Created'
 
host = 'www.google.com';
port = 80;
 
try:
    remote_ip = socket.gethostbyname( host )
 
except socket.gaierror:
    #could not resolve
    print 'Hostname could not be resolved. Exiting'
    sys.exit()
 
#Connect to remote server
s.connect((remote_ip , port))
 
print 'Socket Connected to ' + host + ' on ip ' + remote_ip
 
#Send some data to remote server
message = "GET / HTTP/1.1\r\n\r\n"
 
try :
    #Set the whole string
    s.sendall(message)
except socket.error:
    #Send failed
    print 'Send failed'
    sys.exit()
 
print 'Message send successfully'
 
#Now receive data
reply = s.recv(4096)
 
print reply

output 

Socket Created
Socket Connected to www.google.com on ip 125.235.30.157
Message send successfully
HTTP/1.1 302 Found
Cache-Control: private
Content-Type: text/html; charset=UTF-8
Location: http://www.google.com.vn/?gfe_rd=cr&ei=Cd_gU72uKIiK8Qe6koDICQ
Content-Length: 262
Date: Tue, 05 Aug 2014 13:41:29 GMT
Server: GFE/2.0
Alternate-Protocol: 80:quic

<HTML><HEAD><meta http-equiv="content-type" content="text/html;charset=utf-8">
<TITLE>302 Moved</TITLE></HEAD><BODY>
<H1>302 Moved</H1>
The document has moved
<A HREF="http://www.google.com.vn/?gfe_rd=cr&amp;ei=Cd_gU72uKIiK8Qe6koDICQ">here</A>.
</BODY></HTML>

httplib and urllib2(with addinfourl)

  • The basic class for processing http is httplib package
  • urllib2 is a Python module that can be used for fetching URLs. It defines functions and classes to help with URL actions (basic and digestauthentication, redirections, cookies, etc). urllib2 handle object types below: 
    default_classes = [ProxyHandler, UnknownHandler, HTTPHandler,
                       HTTPDefaultErrorHandler, HTTPRedirectHandler,
                       FTPHandler, FileHandler, HTTPErrorProcessor]

    With http, urllib2 using httplib for sending and receiving data

urllib2.urlopen return addinfourl and debug send data in request

  • addinfourl have basic methods below:
    class addinfourl(addbase):
    """class to add info() and geturl() methods to an open file."""
 
    def __init__(self, fp, headers, url, code=None):
        addbase.__init__(self, fp)
        self.headers = headers
        self.url = url
        self.code = code
 
    def info(self):
        return self.headers
 
    def getcode(self):
        return self.code
 
    def geturl(self):
        return self.url
  • debug send data in request: Edit send function in httplib.HTTPConnection:
        def send(self, data):
        """Send `data' to the server."""
...............................
        else:
            self.sock.sendall(data)
            print data

    and other methods the same read and write file such as read, readlines, next, close 

    class addbase:
    """Base class for addinfo and addclosehook."""
 
    def __init__(self, fp):
        self.fp = fp
        self.read = self.fp.read
        self.readline = self.fp.readline
        if hasattr(self.fp, "readlines"): self.readlines = self.fp.readlines
        if hasattr(self.fp, "fileno"):
            self.fileno = self.fp.fileno
        else:
            self.fileno = lambda: None
        if hasattr(self.fp, "__iter__"):
            self.__iter__ = self.fp.__iter__
            if hasattr(self.fp, "next"):
                self.next = self.fp.next
 
    def __repr__(self):
        return '<%s at %r whose fp = %r>' % (self.__class__.__name__,
                                             id(self), self.fp)
 
    def close(self):
        self.read = None
        self.readline = None
        self.readlines = None
        self.fileno = None
        if self.fp: self.fp.close()
        self.fp = None

simple request(GET)

  • Simple request with httplib
    import httplib
url = 'www.python.org'
conn = httplib.HTTPConnection(url)
conn.request('GET', '')
res = conn.getresponse()
print res.status, res.reason
print res.read()

    ⇒ output: 

    301 Moved Permanently
  • Simple request with urllib2:
    import urllib2
response = urllib2.urlopen('http://www.python.org')
print response.read()

    or 

    import urllib2
req = urllib2.Request('http://www.python.org')
response = urllib2.urlopen(req)
print response.read()

    ⇒ response is urllib.addinfourl object and output are full html data

simple post

  • POST with httplib
    import httplib, urllib
params = urllib.urlencode({'@number': 12524, '@type': 'issue', '@action': 'show'})
headers = {"Content-type": "application/x-www-form-urlencoded",
           "Accept": "text/plain"}
conn = httplib.HTTPConnection("bugs.python.org")
conn.request("POST", "", params, headers)
response = conn.getresponse()
print response.status, response.reason
data = response.read()
print data
conn.close()

    output: 

    302 Found
Redirecting to <a href="http://bugs.python.org/issue12524">http://bugs.python.org/issue12524</a>
  • POST with urllib2
    import urllib, urllib2
params = urllib.urlencode({'@number': 12524, '@type': 'issue', '@action': 'show'})
headers = {"Content-type": "application/x-www-form-urlencoded",
           "Accept": "text/plain"}
req = urllib2.Request("http://bugs.python.org", data = params, headers = headers)
response = urllib2.urlopen(req)
data = response.read()
print data

    ⇒ Output with full html data

urllib2.build_opener(replace urlopen) and post with cookies

  • Using build_opener for sending request:
    import urllib2
 
opener = urllib2.build_opener(urllib2.HTTPHandler()) 
response = opener.open('http://www.python.org')
print response.read()

    and for sending post: 

    import urllib, urllib2
params = urllib.urlencode({'@number': 12524, '@type': 'issue', '@action': 'show'})
headers = {"Content-type": "application/x-www-form-urlencoded",
           "Accept": "text/plain"}
req = urllib2.Request("http://bugs.python.org", data = params, headers = headers)
 
opener = urllib2.build_opener(urllib2.HTTPHandler()) 
response = opener.open(req)
print 'headers:', response.info()
print response.read()

    output: 

    headers: Date: Wed, 06 Aug 2014 04:01:39 GMT
Server: BaseHTTP/0.3 Python/2.6.6
Expires: Wed, 06 Aug 2014 04:01:38 GMT
Content-Type: text/html; charset=utf-8
Cache-Control: no-cache
Via: 1.1 bugs.python.org
Connection: close
Transfer-Encoding: chunked
  • Send normal login to OpenCart:
    import urllib, urllib2
 
loginurl = 'http://dacsan.babies.vn/index.php?route=account/login'
params = urllib.urlencode({'email': '[email protected]', 'password':'8941362', 'redirect':'http://dacsan.babies.vn/index.php?route=account/account'})
headers = {"Content-type": "application/x-www-form-urlencoded",
           "Accept": "text/plain"}
 
req = urllib2.Request(loginurl, data = params, headers = headers)
response = urllib2.urlopen(req)
 
data = response.read()
with open('loginresult.html', 'wb') as f:
    f.write(data)

    Debug in httplib.HTTPConnection send: 

    POST /index.php?route=account/login HTTP/1.1
Accept-Encoding: identity
Content-Length: 122
Host: dacsan.babies.vn
Accept: text/plain
User-Agent: Python-urllib/2.7
Connection: close
Content-Type: application/x-www-form-urlencoded

redirect=http%3A%2F%2Fdacsan.babies.vn%2Findex.php%3Froute%3Daccount%2Faccount&password=8941362&email=itanhchi%40yahoo.com
GET /index.php?route=account/account HTTP/1.1
Accept-Encoding: identity
Host: dacsan.babies.vn
Connection: close
Accept: text/plain
User-Agent: Python-urllib/2.7


GET /index.php?route=account/login HTTP/1.1
Accept-Encoding: identity
Host: dacsan.babies.vn
Connection: close
Accept: text/plain
User-Agent: Python-urllib/2.7
  • Send login to OpenCart with cookies header:
    import urllib, urllib2
 
loginurl = 'http://dacsan.babies.vn/index.php?route=account/login'
params = urllib.urlencode({'email': '[email protected]', 'password':'8941362', 'redirect':'http://dacsan.babies.vn/index.php?route=account/account'})
headers = {"Content-type": "application/x-www-form-urlencoded",
           "Accept": "text/plain"}
req = urllib2.Request(loginurl, data = params, headers = headers)
 
opener = urllib2.build_opener(urllib2.HTTPCookieProcessor()) 
response = opener.open(req)
 
data = response.read()
with open('loginresult.html', 'wb') as f:
    f.write(data)

    Debug in httplib.HTTPConnection send: 

    POST /index.php?route=account/login HTTP/1.1
Accept-Encoding: identity
Content-Length: 122
Host: dacsan.babies.vn
Accept: text/plain
User-Agent: Python-urllib/2.7
Connection: close
Content-Type: application/x-www-form-urlencoded

redirect=http%3A%2F%2Fdacsan.babies.vn%2Findex.php%3Froute%3Daccount%2Faccount&password=8941362&email=itanhchi%40yahoo.com
GET /index.php?route=account/account HTTP/1.1
Accept-Encoding: identity
Host: dacsan.babies.vn
Cookie: currency=VND; language=vn; PHPSESSID=h1mr19s902mekq25ubccsn4k22
Connection: close
Accept: text/plain
User-Agent: Python-urllib/2.7
import urllib, urllib2
import cookielib
 
loginurl = 'http://dacsan.babies.vn/index.php?route=account/login'
params = urllib.urlencode({'email': '[email protected]', 'password':'8941362', 'redirect':'http://dacsan.babies.vn/index.php?route=account/account'})
headers = {"Content-type": "application/x-www-form-urlencoded",
           "Accept": "text/plain"}
req = urllib2.Request(loginurl, data = params, headers = headers)
 
cj = cookielib.CookieJar() 
opener = urllib2.build_opener(urllib2.HTTPCookieProcessor(cj)) 
response = opener.open(req)
 
for cookie in cj:
    print cookie.name, cookie.value
data = response.read()
with open('loginresult.html', 'wb') as f:
    f.write(data)

output: 

currency VND
language vn
PHPSESSID 01eg0u7uf5bm3r01h6pnrv3q33

Post multipart form data

post with encode_multipart_formdata

Post with httplib 

import httplib, mimetypes
 
def post_multipart(host, selector, fields, files):
    """
    Post fields and files to an http host as multipart/form-data.
    fields is a sequence of (name, value) elements for regular form fields.
    files is a sequence of (name, filename, value) elements for data to be uploaded as files
    Return the server's response page.
    """
    content_type, body = encode_multipart_formdata(fields, files)
    h = httplib.HTTP(host)
    h.putrequest('POST', selector)
    h.putheader('content-type', content_type)
    h.putheader('content-length', str(len(body)))
    h.endheaders()
    h.send(body)
    errcode, errmsg, headers = h.getreply()
    return h.file.read()
 
def encode_multipart_formdata(fields, files):
    """
    fields is a sequence of (name, value) elements for regular form fields.
    files is a sequence of (name, filename, value) elements for data to be uploaded as files
    Return (content_type, body) ready for httplib.HTTP instance
    """
    BOUNDARY = '----------ThIs_Is_tHe_bouNdaRY_$'
    CRLF = '\r\n'
    L = []
    for (key, value) in fields:
        L.append('--' + BOUNDARY)
        L.append('Content-Disposition: form-data; name="%s"' % key)
        L.append('')
        L.append(value)
    for (key, filename, value) in files:
        L.append('--' + BOUNDARY)
        L.append('Content-Disposition: form-data; name="%s"; filename="%s"' % (key, filename))
        L.append('Content-Type: %s' % get_content_type(filename))
        L.append('')
        L.append(value)
    L.append('--' + BOUNDARY + '--')
    L.append('')
    body = CRLF.join(L)
    content_type = 'multipart/form-data; boundary=%s' % BOUNDARY
    return content_type, body
 
def get_content_type(filename):
    return mimetypes.guess_type(filename)[0] or 'application/octet-stream'

fix encode_multipart_formdata for posting binary file

def encode_multipart_formdata(fields, files):
    """
    fields is a sequence of (name, value) elements for regular form fields.
    files is a sequence of (name, filename, value) elements for data to be uploaded as files
    Return (content_type, body) ready for httplib.HTTP instance
    """    
 
    buf = StringIO()
    boundary = mimetools.choose_boundary()            
    for (key, value) in fields:
        buf.write('--%s\r\n' % boundary)
        buf.write('Content-Disposition: form-data; name="%s"' % key)
        buf.write('\r\n\r\n' + value + '\r\n')
    for (key, filename, value) in files:
        contenttype = mimetypes.guess_type(filename)[0] or 'application/octet-stream'
        buf.write('--%s\r\n' % boundary)
        buf.write('Content-Disposition: form-data; name="%s"; filename="%s"\r\n' % (key, filename))
        buf.write('Content-Type: %s\r\n' % contenttype)
        buf.write('\r\n' + value + '\r\n')
    buf.write('--' + boundary + '--\r\n\r\n')
    buf = buf.getvalue()
    content_type = 'multipart/form-data; boundary=%s' % boundary
    return content_type, buf

Post using MultipartPostHandler

  • Install:
    pip install MultipartPostHandler
  • example:
    import MultipartPostHandler, urllib2, cookielib
 
cookies = cookielib.CookieJar()
opener = urllib2.build_opener(urllib2.HTTPCookieProcessor(cookies),
                                MultipartPostHandler.MultipartPostHandler)
params = { "username" : "bob", "password" : "riviera",
           "file" : open("filename", "rb") }
opener.open("http://wwww.bobsite.com/upload/", params)

twisted internet

Simple Request

from sys import argv
from pprint import pformat
 
from twisted.internet.task import react
from twisted.web.client import Agent, readBody
from twisted.web.http_headers import Headers
 
 
def cbRequest(response):
    print 'Response version:', response.version
    print 'Response code:', response.code
    print 'Response phrase:', response.phrase
    print 'Response headers:'
    print pformat(list(response.headers.getAllRawHeaders()))
    d = readBody(response)
    d.addCallback(cbBody)
    return d
 
def cbBody(body):
    print 'Response body:'
    print body
 
def main(reactor, url=b"http://www.google.com/"):
    agent = Agent(reactor)
    d = agent.request(
        'GET', url,
        Headers({'User-Agent': ['Twisted Web Client Example']}),
        None)
    d.addCallback(cbRequest)
    return d
 
react(main, argv[1:])
python/internet.txt · Last modified: 2022/10/29 16:15 by 127.0.0.1