User Tools

Site Tools


python:internet

Python Internet

Python Socket

Echo Server and Client

Simple Echo Server and Client

  • Echo Server
    # Echo server program
    import socket
     
    HOST = ''                 # Symbolic name meaning all available interfaces
    PORT = 50007              # Arbitrary non-privileged port
    s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
    s.bind((HOST, PORT))
    s.listen(1)
    print 'server is listening'
    conn, addr = s.accept()
    print 'Connected by', addr
    while 1:
        data = conn.recv(1024)
        if not data: break
        conn.sendall(data)
    conn.close()

    output:

    server is listening
  • Echo Client
    # Echo client program
    import socket
     
    HOST = '127.0.0.1'    # The remote host
    PORT = 50007              # The same port as used by the server
    s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
    s.connect((HOST, PORT))
    s.sendall('Hello, world')
    data = s.recv(1024)
    s.close()
    print 'Received', repr(data)

    server output:

    server is listening
    Connected by ('127.0.0.1', 56354)

    client output:

    Received 'Hello, world'

Custom Echo Server for multiple client

The above example only service for 1 client, after the server receive connection and send data to client, the echoserver program will end. The below example will service for multiple clients

  • Echo server
    # Echo server program
    import socket
     
    HOST = ''                 # Symbolic name meaning all available interfaces
    PORT = 50007              # Arbitrary non-privileged port
    s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
    s.bind((HOST, PORT))
    s.listen(10)
    print 'server is listening'
    while 1:
        conn, addr = s.accept()
        print 'Connected by', addr
        data = conn.recv(1024)
        if not data: break
        conn.sendall(data)
    conn.close()

Connect to google with socket and receive data

#Socket client example in python
 
import socket   #for sockets
import sys  #for exit
 
#create an INET, STREAMing socket
try:
    s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
except socket.error:
    print 'Failed to create socket'
    sys.exit()
 
print 'Socket Created'
 
host = 'www.google.com';
port = 80;
 
try:
    remote_ip = socket.gethostbyname( host )
 
except socket.gaierror:
    #could not resolve
    print 'Hostname could not be resolved. Exiting'
    sys.exit()
 
#Connect to remote server
s.connect((remote_ip , port))
 
print 'Socket Connected to ' + host + ' on ip ' + remote_ip
 
#Send some data to remote server
message = "GET / HTTP/1.1\r\n\r\n"
 
try :
    #Set the whole string
    s.sendall(message)
except socket.error:
    #Send failed
    print 'Send failed'
    sys.exit()
 
print 'Message send successfully'
 
#Now receive data
reply = s.recv(4096)
 
print reply

output

Socket Created
Socket Connected to www.google.com on ip 125.235.30.157
Message send successfully
HTTP/1.1 302 Found
Cache-Control: private
Content-Type: text/html; charset=UTF-8
Location: http://www.google.com.vn/?gfe_rd=cr&ei=Cd_gU72uKIiK8Qe6koDICQ
Content-Length: 262
Date: Tue, 05 Aug 2014 13:41:29 GMT
Server: GFE/2.0
Alternate-Protocol: 80:quic

<HTML><HEAD><meta http-equiv="content-type" content="text/html;charset=utf-8">
<TITLE>302 Moved</TITLE></HEAD><BODY>
<H1>302 Moved</H1>
The document has moved
<A HREF="http://www.google.com.vn/?gfe_rd=cr&amp;ei=Cd_gU72uKIiK8Qe6koDICQ">here</A>.
</BODY></HTML>

httplib and urllib2(with addinfourl)

  • The basic class for processing http is httplib package
  • urllib2 is a Python module that can be used for fetching URLs. It defines functions and classes to help with URL actions (basic and digestauthentication, redirections, cookies, etc). urllib2 handle object types below:
    default_classes = [ProxyHandler, UnknownHandler, HTTPHandler,
                           HTTPDefaultErrorHandler, HTTPRedirectHandler,
                           FTPHandler, FileHandler, HTTPErrorProcessor]

    With http, urllib2 using httplib for sending and receiving data

urllib2.urlopen return addinfourl and debug send data in request

  • addinfourl have basic methods below:
    class addinfourl(addbase):
        """class to add info() and geturl() methods to an open file."""
     
        def __init__(self, fp, headers, url, code=None):
            addbase.__init__(self, fp)
            self.headers = headers
            self.url = url
            self.code = code
     
        def info(self):
            return self.headers
     
        def getcode(self):
            return self.code
     
        def geturl(self):
            return self.url
  • debug send data in request: Edit send function in httplib.HTTPConnection:
        def send(self, data):
            """Send `data' to the server."""
    ...............................
            else:
                self.sock.sendall(data)
                print data

    and other methods the same read and write file such as read, readlines, next, close

    class addbase:
        """Base class for addinfo and addclosehook."""
     
        def __init__(self, fp):
            self.fp = fp
            self.read = self.fp.read
            self.readline = self.fp.readline
            if hasattr(self.fp, "readlines"): self.readlines = self.fp.readlines
            if hasattr(self.fp, "fileno"):
                self.fileno = self.fp.fileno
            else:
                self.fileno = lambda: None
            if hasattr(self.fp, "__iter__"):
                self.__iter__ = self.fp.__iter__
                if hasattr(self.fp, "next"):
                    self.next = self.fp.next
     
        def __repr__(self):
            return '<%s at %r whose fp = %r>' % (self.__class__.__name__,
                                                 id(self), self.fp)
     
        def close(self):
            self.read = None
            self.readline = None
            self.readlines = None
            self.fileno = None
            if self.fp: self.fp.close()
            self.fp = None

simple request(GET)

  • Simple request with httplib
    import httplib
    url = 'www.python.org'
    conn = httplib.HTTPConnection(url)
    conn.request('GET', '')
    res = conn.getresponse()
    print res.status, res.reason
    print res.read()

    ⇒ output:

    301 Moved Permanently
  • Simple request with urllib2:
    import urllib2
    response = urllib2.urlopen('http://www.python.org')
    print response.read()

    or

    import urllib2
    req = urllib2.Request('http://www.python.org')
    response = urllib2.urlopen(req)
    print response.read()

    ⇒ response is urllib.addinfourl object and output are full html data

simple post

  • POST with httplib
    import httplib, urllib
    params = urllib.urlencode({'@number': 12524, '@type': 'issue', '@action': 'show'})
    headers = {"Content-type": "application/x-www-form-urlencoded",
               "Accept": "text/plain"}
    conn = httplib.HTTPConnection("bugs.python.org")
    conn.request("POST", "", params, headers)
    response = conn.getresponse()
    print response.status, response.reason
    data = response.read()
    print data
    conn.close()

    output:

    302 Found
    Redirecting to <a href="http://bugs.python.org/issue12524">http://bugs.python.org/issue12524</a>
  • POST with urllib2
    import urllib, urllib2
    params = urllib.urlencode({'@number': 12524, '@type': 'issue', '@action': 'show'})
    headers = {"Content-type": "application/x-www-form-urlencoded",
               "Accept": "text/plain"}
    req = urllib2.Request("http://bugs.python.org", data = params, headers = headers)
    response = urllib2.urlopen(req)
    data = response.read()
    print data

    ⇒ Output with full html data

urllib2.build_opener(replace urlopen) and post with cookies

  • Using build_opener for sending request:
    import urllib2
     
    opener = urllib2.build_opener(urllib2.HTTPHandler()) 
    response = opener.open('http://www.python.org')
    print response.read()

    and for sending post:

    import urllib, urllib2
    params = urllib.urlencode({'@number': 12524, '@type': 'issue', '@action': 'show'})
    headers = {"Content-type": "application/x-www-form-urlencoded",
               "Accept": "text/plain"}
    req = urllib2.Request("http://bugs.python.org", data = params, headers = headers)
     
    opener = urllib2.build_opener(urllib2.HTTPHandler()) 
    response = opener.open(req)
    print 'headers:', response.info()
    print response.read()

    output:

    headers: Date: Wed, 06 Aug 2014 04:01:39 GMT
    Server: BaseHTTP/0.3 Python/2.6.6
    Expires: Wed, 06 Aug 2014 04:01:38 GMT
    Content-Type: text/html; charset=utf-8
    Cache-Control: no-cache
    Via: 1.1 bugs.python.org
    Connection: close
    Transfer-Encoding: chunked
  • Send normal login to OpenCart:
    import urllib, urllib2
     
    loginurl = 'http://dacsan.babies.vn/index.php?route=account/login'
    params = urllib.urlencode({'email': '[email protected]', 'password':'8941362', 'redirect':'http://dacsan.babies.vn/index.php?route=account/account'})
    headers = {"Content-type": "application/x-www-form-urlencoded",
               "Accept": "text/plain"}
     
    req = urllib2.Request(loginurl, data = params, headers = headers)
    response = urllib2.urlopen(req)
     
    data = response.read()
    with open('loginresult.html', 'wb') as f:
        f.write(data)

    Debug in httplib.HTTPConnection send:

    POST /index.php?route=account/login HTTP/1.1
    Accept-Encoding: identity
    Content-Length: 122
    Host: dacsan.babies.vn
    Accept: text/plain
    User-Agent: Python-urllib/2.7
    Connection: close
    Content-Type: application/x-www-form-urlencoded
    
    redirect=http%3A%2F%2Fdacsan.babies.vn%2Findex.php%3Froute%3Daccount%2Faccount&password=8941362&email=itanhchi%40yahoo.com
    GET /index.php?route=account/account HTTP/1.1
    Accept-Encoding: identity
    Host: dacsan.babies.vn
    Connection: close
    Accept: text/plain
    User-Agent: Python-urllib/2.7
    
    
    GET /index.php?route=account/login HTTP/1.1
    Accept-Encoding: identity
    Host: dacsan.babies.vn
    Connection: close
    Accept: text/plain
    User-Agent: Python-urllib/2.7
  • Send login to OpenCart with cookies header:
    import urllib, urllib2
     
    loginurl = 'http://dacsan.babies.vn/index.php?route=account/login'
    params = urllib.urlencode({'email': '[email protected]', 'password':'8941362', 'redirect':'http://dacsan.babies.vn/index.php?route=account/account'})
    headers = {"Content-type": "application/x-www-form-urlencoded",
               "Accept": "text/plain"}
    req = urllib2.Request(loginurl, data = params, headers = headers)
     
    opener = urllib2.build_opener(urllib2.HTTPCookieProcessor()) 
    response = opener.open(req)
     
    data = response.read()
    with open('loginresult.html', 'wb') as f:
        f.write(data)

    Debug in httplib.HTTPConnection send:

    POST /index.php?route=account/login HTTP/1.1
    Accept-Encoding: identity
    Content-Length: 122
    Host: dacsan.babies.vn
    Accept: text/plain
    User-Agent: Python-urllib/2.7
    Connection: close
    Content-Type: application/x-www-form-urlencoded
    
    redirect=http%3A%2F%2Fdacsan.babies.vn%2Findex.php%3Froute%3Daccount%2Faccount&password=8941362&email=itanhchi%40yahoo.com
    GET /index.php?route=account/account HTTP/1.1
    Accept-Encoding: identity
    Host: dacsan.babies.vn
    Cookie: currency=VND; language=vn; PHPSESSID=h1mr19s902mekq25ubccsn4k22
    Connection: close
    Accept: text/plain
    User-Agent: Python-urllib/2.7
import urllib, urllib2
import cookielib
 
loginurl = 'http://dacsan.babies.vn/index.php?route=account/login'
params = urllib.urlencode({'email': '[email protected]', 'password':'8941362', 'redirect':'http://dacsan.babies.vn/index.php?route=account/account'})
headers = {"Content-type": "application/x-www-form-urlencoded",
           "Accept": "text/plain"}
req = urllib2.Request(loginurl, data = params, headers = headers)
 
cj = cookielib.CookieJar() 
opener = urllib2.build_opener(urllib2.HTTPCookieProcessor(cj)) 
response = opener.open(req)
 
for cookie in cj:
    print cookie.name, cookie.value
data = response.read()
with open('loginresult.html', 'wb') as f:
    f.write(data)

output:

currency VND
language vn
PHPSESSID 01eg0u7uf5bm3r01h6pnrv3q33

Post multipart form data

post with encode_multipart_formdata

Post with httplib

import httplib, mimetypes
 
def post_multipart(host, selector, fields, files):
    """
    Post fields and files to an http host as multipart/form-data.
    fields is a sequence of (name, value) elements for regular form fields.
    files is a sequence of (name, filename, value) elements for data to be uploaded as files
    Return the server's response page.
    """
    content_type, body = encode_multipart_formdata(fields, files)
    h = httplib.HTTP(host)
    h.putrequest('POST', selector)
    h.putheader('content-type', content_type)
    h.putheader('content-length', str(len(body)))
    h.endheaders()
    h.send(body)
    errcode, errmsg, headers = h.getreply()
    return h.file.read()
 
def encode_multipart_formdata(fields, files):
    """
    fields is a sequence of (name, value) elements for regular form fields.
    files is a sequence of (name, filename, value) elements for data to be uploaded as files
    Return (content_type, body) ready for httplib.HTTP instance
    """
    BOUNDARY = '----------ThIs_Is_tHe_bouNdaRY_$'
    CRLF = '\r\n'
    L = []
    for (key, value) in fields:
        L.append('--' + BOUNDARY)
        L.append('Content-Disposition: form-data; name="%s"' % key)
        L.append('')
        L.append(value)
    for (key, filename, value) in files:
        L.append('--' + BOUNDARY)
        L.append('Content-Disposition: form-data; name="%s"; filename="%s"' % (key, filename))
        L.append('Content-Type: %s' % get_content_type(filename))
        L.append('')
        L.append(value)
    L.append('--' + BOUNDARY + '--')
    L.append('')
    body = CRLF.join(L)
    content_type = 'multipart/form-data; boundary=%s' % BOUNDARY
    return content_type, body
 
def get_content_type(filename):
    return mimetypes.guess_type(filename)[0] or 'application/octet-stream'

fix encode_multipart_formdata for posting binary file

def encode_multipart_formdata(fields, files):
    """
    fields is a sequence of (name, value) elements for regular form fields.
    files is a sequence of (name, filename, value) elements for data to be uploaded as files
    Return (content_type, body) ready for httplib.HTTP instance
    """    
 
    buf = StringIO()
    boundary = mimetools.choose_boundary()            
    for (key, value) in fields:
        buf.write('--%s\r\n' % boundary)
        buf.write('Content-Disposition: form-data; name="%s"' % key)
        buf.write('\r\n\r\n' + value + '\r\n')
    for (key, filename, value) in files:
        contenttype = mimetypes.guess_type(filename)[0] or 'application/octet-stream'
        buf.write('--%s\r\n' % boundary)
        buf.write('Content-Disposition: form-data; name="%s"; filename="%s"\r\n' % (key, filename))
        buf.write('Content-Type: %s\r\n' % contenttype)
        buf.write('\r\n' + value + '\r\n')
    buf.write('--' + boundary + '--\r\n\r\n')
    buf = buf.getvalue()
    content_type = 'multipart/form-data; boundary=%s' % boundary
    return content_type, buf

Post using MultipartPostHandler

  • Install:
    pip install MultipartPostHandler
  • example:
    import MultipartPostHandler, urllib2, cookielib
     
    cookies = cookielib.CookieJar()
    opener = urllib2.build_opener(urllib2.HTTPCookieProcessor(cookies),
                                    MultipartPostHandler.MultipartPostHandler)
    params = { "username" : "bob", "password" : "riviera",
               "file" : open("filename", "rb") }
    opener.open("http://wwww.bobsite.com/upload/", params)

twisted internet

Simple Request

from sys import argv
from pprint import pformat
 
from twisted.internet.task import react
from twisted.web.client import Agent, readBody
from twisted.web.http_headers import Headers
 
 
def cbRequest(response):
    print 'Response version:', response.version
    print 'Response code:', response.code
    print 'Response phrase:', response.phrase
    print 'Response headers:'
    print pformat(list(response.headers.getAllRawHeaders()))
    d = readBody(response)
    d.addCallback(cbBody)
    return d
 
def cbBody(body):
    print 'Response body:'
    print body
 
def main(reactor, url=b"http://www.google.com/"):
    agent = Agent(reactor)
    d = agent.request(
        'GET', url,
        Headers({'User-Agent': ['Twisted Web Client Example']}),
        None)
    d.addCallback(cbRequest)
    return d
 
react(main, argv[1:])
python/internet.txt · Last modified: 2022/10/29 16:15 by 127.0.0.1