302 Moved

====== Python Internet ====== ===== Python Socket ===== refer: https://docs.python.org/2/library/socket.html ==== Echo Server and Client ==== === Simple Echo Server and Client === * Echo Server


# Echo server program
import socket
 
HOST = ''                 # Symbolic name meaning all available interfaces
PORT = 50007              # Arbitrary non-privileged port
s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
s.bind((HOST, PORT))
s.listen(1)
print 'server is listening'
conn, addr = s.accept()
print 'Connected by', addr
while 1:
    data = conn.recv(1024)
    if not data: break
    conn.sendall(data)
conn.close()

output:


server is listening

* Echo Client


# Echo client program
import socket

HOST = '127.0.0.1'    # The remote host
PORT = 50007              # The same port as used by the server
s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
s.connect((HOST, PORT))
s.sendall('Hello, world')
data = s.recv(1024)
s.close()
print 'Received', repr(data)

server output:


server is listening
Connected by ('127.0.0.1', 56354)

client output:


Received 'Hello, world'

=== Custom Echo Server for multiple client === The above example only service for 1 client, after the server receive connection and send data to client, the echoserver program will end. The below example will service for multiple clients * Echo server


# Echo server program
import socket
 
HOST = ''                 # Symbolic name meaning all available interfaces
PORT = 50007              # Arbitrary non-privileged port
s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
s.bind((HOST, PORT))
s.listen(10)
print 'server is listening'
while 1:
    conn, addr = s.accept()
    print 'Connected by', addr
    data = conn.recv(1024)
    if not data: break
    conn.sendall(data)
conn.close()

==== Connect to google with socket and receive data ====


#Socket client example in python
 
import socket   #for sockets
import sys  #for exit
 
#create an INET, STREAMing socket
try:
    s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
except socket.error:
    print 'Failed to create socket'
    sys.exit()
     
print 'Socket Created'
 
host = 'www.google.com';
port = 80;
 
try:
    remote_ip = socket.gethostbyname( host )
 
except socket.gaierror:
    #could not resolve
    print 'Hostname could not be resolved. Exiting'
    sys.exit()
 
#Connect to remote server
s.connect((remote_ip , port))
 
print 'Socket Connected to ' + host + ' on ip ' + remote_ip
 
#Send some data to remote server
message = "GET / HTTP/1.1\r\n\r\n"
 
try :
    #Set the whole string
    s.sendall(message)
except socket.error:
    #Send failed
    print 'Send failed'
    sys.exit()
 
print 'Message send successfully'
 
#Now receive data
reply = s.recv(4096)
 
print reply

output


Socket Created
Socket Connected to www.google.com on ip 125.235.30.157
Message send successfully
HTTP/1.1 302 Found
Cache-Control: private
Content-Type: text/html; charset=UTF-8
Location: http://www.google.com.vn/?gfe_rd=cr&ei=Cd_gU72uKIiK8Qe6koDICQ
Content-Length: 262
Date: Tue, 05 Aug 2014 13:41:29 GMT
Server: GFE/2.0
Alternate-Protocol: 80:quic


302 Moved
302 Moved
The document has moved
here.

===== httplib and urllib2(with addinfourl) ===== * The basic class for processing http is httplib package * urllib2 is a Python module that can be used for fetching URLs. It defines functions and classes to help with URL actions (basic and digestauthentication, redirections, cookies, etc). urllib2 handle object types below:


default_classes = [ProxyHandler, UnknownHandler, HTTPHandler,
                       HTTPDefaultErrorHandler, HTTPRedirectHandler,
                       FTPHandler, FileHandler, HTTPErrorProcessor]

With http, urllib2 using httplib for sending and receiving data ==== urllib2.urlopen return addinfourl and debug send data in request ===== * addinfourl have basic methods below:


class addinfourl(addbase):
    """class to add info() and geturl() methods to an open file."""

    def __init__(self, fp, headers, url, code=None):
        addbase.__init__(self, fp)
        self.headers = headers
        self.url = url
        self.code = code

    def info(self):
        return self.headers

    def getcode(self):
        return self.code

    def geturl(self):
        return self.url

* debug send data in request: Edit send function in httplib.HTTPConnection:


    def send(self, data):
        """Send `data' to the server."""
...............................
        else:
            self.sock.sendall(data)
            print data

and other methods the same read and write file such as **read, readlines, next, close**


class addbase:
    """Base class for addinfo and addclosehook."""

    def __init__(self, fp):
        self.fp = fp
        self.read = self.fp.read
        self.readline = self.fp.readline
        if hasattr(self.fp, "readlines"): self.readlines = self.fp.readlines
        if hasattr(self.fp, "fileno"):
            self.fileno = self.fp.fileno
        else:
            self.fileno = lambda: None
        if hasattr(self.fp, "__iter__"):
            self.__iter__ = self.fp.__iter__
            if hasattr(self.fp, "next"):
                self.next = self.fp.next

    def __repr__(self):
        return '<%s at %r whose fp = %r>' % (self.__class__.__name__,
                                             id(self), self.fp)

    def close(self):
        self.read = None
        self.readline = None
        self.readlines = None
        self.fileno = None
        if self.fp: self.fp.close()
        self.fp = None

==== simple request(GET) ==== * Simple request with httplib


import httplib
url = 'www.python.org'
conn = httplib.HTTPConnection(url)
conn.request('GET', '')
res = conn.getresponse()
print res.status, res.reason
print res.read()

=> output:


301 Moved Permanently

* Simple request with urllib2:


import urllib2
response = urllib2.urlopen('http://www.python.org')
print response.read()


import urllib2
req = urllib2.Request('http://www.python.org')
response = urllib2.urlopen(req)
print response.read()

=> response is **urllib.addinfourl** object and output are full html data ==== simple post ==== * POST with httplib


import httplib, urllib
params = urllib.urlencode({'@number': 12524, '@type': 'issue', '@action': 'show'})
headers = {"Content-type": "application/x-www-form-urlencoded",
           "Accept": "text/plain"}
conn = httplib.HTTPConnection("bugs.python.org")
conn.request("POST", "", params, headers)
response = conn.getresponse()
print response.status, response.reason
data = response.read()
print data
conn.close()

output:


302 Found
Redirecting to http://bugs.python.org/issue12524

* POST with urllib2


import urllib, urllib2
params = urllib.urlencode({'@number': 12524, '@type': 'issue', '@action': 'show'})
headers = {"Content-type": "application/x-www-form-urlencoded",
           "Accept": "text/plain"}
req = urllib2.Request("http://bugs.python.org", data = params, headers = headers)
response = urllib2.urlopen(req)
data = response.read()
print data

=> Output with full html data ==== urllib2.build_opener(replace urlopen) and post with cookies ==== * Using build_opener for sending request:


import urllib2

opener = urllib2.build_opener(urllib2.HTTPHandler()) 
response = opener.open('http://www.python.org')
print response.read()

and for sending post:


import urllib, urllib2
params = urllib.urlencode({'@number': 12524, '@type': 'issue', '@action': 'show'})
headers = {"Content-type": "application/x-www-form-urlencoded",
           "Accept": "text/plain"}
req = urllib2.Request("http://bugs.python.org", data = params, headers = headers)

opener = urllib2.build_opener(urllib2.HTTPHandler()) 
response = opener.open(req)
print 'headers:', response.info()
print response.read()

output:


headers: Date: Wed, 06 Aug 2014 04:01:39 GMT
Server: BaseHTTP/0.3 Python/2.6.6
Expires: Wed, 06 Aug 2014 04:01:38 GMT
Content-Type: text/html; charset=utf-8
Cache-Control: no-cache
Via: 1.1 bugs.python.org
Connection: close
Transfer-Encoding: chunked

* Send normal login to OpenCart:


import urllib, urllib2

loginurl = 'http://dacsan.babies.vn/index.php?route=account/login'
params = urllib.urlencode({'email': 'itanhchi@yahoo.com', 'password':'8941362', 'redirect':'http://dacsan.babies.vn/index.php?route=account/account'})
headers = {"Content-type": "application/x-www-form-urlencoded",
           "Accept": "text/plain"}

req = urllib2.Request(loginurl, data = params, headers = headers)
response = urllib2.urlopen(req)

data = response.read()
with open('loginresult.html', 'wb') as f:
    f.write(data)

Debug in httplib.HTTPConnection send:


POST /index.php?route=account/login HTTP/1.1
Accept-Encoding: identity
Content-Length: 122
Host: dacsan.babies.vn
Accept: text/plain
User-Agent: Python-urllib/2.7
Connection: close
Content-Type: application/x-www-form-urlencoded

redirect=http%3A%2F%2Fdacsan.babies.vn%2Findex.php%3Froute%3Daccount%2Faccount&password=8941362&email=itanhchi%40yahoo.com
GET /index.php?route=account/account HTTP/1.1
Accept-Encoding: identity
Host: dacsan.babies.vn
Connection: close
Accept: text/plain
User-Agent: Python-urllib/2.7


GET /index.php?route=account/login HTTP/1.1
Accept-Encoding: identity
Host: dacsan.babies.vn
Connection: close
Accept: text/plain
User-Agent: Python-urllib/2.7

* Send login to OpenCart with cookies header:


import urllib, urllib2
 
loginurl = 'http://dacsan.babies.vn/index.php?route=account/login'
params = urllib.urlencode({'email': 'itanhchi@yahoo.com', 'password':'8941362', 'redirect':'http://dacsan.babies.vn/index.php?route=account/account'})
headers = {"Content-type": "application/x-www-form-urlencoded",
           "Accept": "text/plain"}
req = urllib2.Request(loginurl, data = params, headers = headers)

opener = urllib2.build_opener(urllib2.HTTPCookieProcessor()) 
response = opener.open(req)

data = response.read()
with open('loginresult.html', 'wb') as f:
    f.write(data)

Debug in httplib.HTTPConnection send:


POST /index.php?route=account/login HTTP/1.1
Accept-Encoding: identity
Content-Length: 122
Host: dacsan.babies.vn
Accept: text/plain
User-Agent: Python-urllib/2.7
Connection: close
Content-Type: application/x-www-form-urlencoded

redirect=http%3A%2F%2Fdacsan.babies.vn%2Findex.php%3Froute%3Daccount%2Faccount&password=8941362&email=itanhchi%40yahoo.com
GET /index.php?route=account/account HTTP/1.1
Accept-Encoding: identity
Host: dacsan.babies.vn
Cookie: currency=VND; language=vn; PHPSESSID=h1mr19s902mekq25ubccsn4k22
Connection: close
Accept: text/plain
User-Agent: Python-urllib/2.7

==== Get Cookie Information ====


import urllib, urllib2
import cookielib
 
loginurl = 'http://dacsan.babies.vn/index.php?route=account/login'
params = urllib.urlencode({'email': 'itanhchi@yahoo.com', 'password':'8941362', 'redirect':'http://dacsan.babies.vn/index.php?route=account/account'})
headers = {"Content-type": "application/x-www-form-urlencoded",
           "Accept": "text/plain"}
req = urllib2.Request(loginurl, data = params, headers = headers)

cj = cookielib.CookieJar() 
opener = urllib2.build_opener(urllib2.HTTPCookieProcessor(cj)) 
response = opener.open(req)

for cookie in cj:
    print cookie.name, cookie.value
data = response.read()
with open('loginresult.html', 'wb') as f:
    f.write(data)

output:


currency VND
language vn
PHPSESSID 01eg0u7uf5bm3r01h6pnrv3q33

==== Post multipart form data ==== === post with encode_multipart_formdata === Post with httplib


import httplib, mimetypes

def post_multipart(host, selector, fields, files):
    """
    Post fields and files to an http host as multipart/form-data.
    fields is a sequence of (name, value) elements for regular form fields.
    files is a sequence of (name, filename, value) elements for data to be uploaded as files
    Return the server's response page.
    """
    content_type, body = encode_multipart_formdata(fields, files)
    h = httplib.HTTP(host)
    h.putrequest('POST', selector)
    h.putheader('content-type', content_type)
    h.putheader('content-length', str(len(body)))
    h.endheaders()
    h.send(body)
    errcode, errmsg, headers = h.getreply()
    return h.file.read()

def encode_multipart_formdata(fields, files):
    """
    fields is a sequence of (name, value) elements for regular form fields.
    files is a sequence of (name, filename, value) elements for data to be uploaded as files
    Return (content_type, body) ready for httplib.HTTP instance
    """
    BOUNDARY = '----------ThIs_Is_tHe_bouNdaRY_$'
    CRLF = '\r\n'
    L = []
    for (key, value) in fields:
        L.append('--' + BOUNDARY)
        L.append('Content-Disposition: form-data; name="%s"' % key)
        L.append('')
        L.append(value)
    for (key, filename, value) in files:
        L.append('--' + BOUNDARY)
        L.append('Content-Disposition: form-data; name="%s"; filename="%s"' % (key, filename))
        L.append('Content-Type: %s' % get_content_type(filename))
        L.append('')
        L.append(value)
    L.append('--' + BOUNDARY + '--')
    L.append('')
    body = CRLF.join(L)
    content_type = 'multipart/form-data; boundary=%s' % BOUNDARY
    return content_type, body

def get_content_type(filename):
    return mimetypes.guess_type(filename)[0] or 'application/octet-stream'

=== fix encode_multipart_formdata for posting binary file ===


def encode_multipart_formdata(fields, files):
    """
    fields is a sequence of (name, value) elements for regular form fields.
    files is a sequence of (name, filename, value) elements for data to be uploaded as files
    Return (content_type, body) ready for httplib.HTTP instance
    """    
    
    buf = StringIO()
    boundary = mimetools.choose_boundary()            
    for (key, value) in fields:
        buf.write('--%s\r\n' % boundary)
        buf.write('Content-Disposition: form-data; name="%s"' % key)
        buf.write('\r\n\r\n' + value + '\r\n')
    for (key, filename, value) in files:
        contenttype = mimetypes.guess_type(filename)[0] or 'application/octet-stream'
        buf.write('--%s\r\n' % boundary)
        buf.write('Content-Disposition: form-data; name="%s"; filename="%s"\r\n' % (key, filename))
        buf.write('Content-Type: %s\r\n' % contenttype)
        buf.write('\r\n' + value + '\r\n')
    buf.write('--' + boundary + '--\r\n\r\n')
    buf = buf.getvalue()
    content_type = 'multipart/form-data; boundary=%s' % boundary
    return content_type, buf

=== Post using MultipartPostHandler === * Install:


pip install MultipartPostHandler

* example:


import MultipartPostHandler, urllib2, cookielib

cookies = cookielib.CookieJar()
opener = urllib2.build_opener(urllib2.HTTPCookieProcessor(cookies),
                                MultipartPostHandler.MultipartPostHandler)
params = { "username" : "bob", "password" : "riviera",
           "file" : open("filename", "rb") }
opener.open("http://wwww.bobsite.com/upload/", params)

===== twisted internet ===== ==== Simple Request ====


from sys import argv
from pprint import pformat

from twisted.internet.task import react
from twisted.web.client import Agent, readBody
from twisted.web.http_headers import Headers


def cbRequest(response):
    print 'Response version:', response.version
    print 'Response code:', response.code
    print 'Response phrase:', response.phrase
    print 'Response headers:'
    print pformat(list(response.headers.getAllRawHeaders()))
    d = readBody(response)
    d.addCallback(cbBody)
    return d

def cbBody(body):
    print 'Response body:'
    print body

def main(reactor, url=b"http://www.google.com/"):
    agent = Agent(reactor)
    d = agent.request(
        'GET', url,
        Headers({'User-Agent': ['Twisted Web Client Example']}),
        None)
    d.addCallback(cbRequest)
    return d

react(main, argv[1:])