jdickinson202

Web Scraping Workshop

Using Requests and Beautiful Soup, with the most recent Beautiful Soup 4 docs.

Getting Started

Install our tools (preferably in a new virtualenv):

pip install beautifulsoup4

jdickinson202

import matplotlib.pyplot as plt
from matplotlib.collections import PatchCollection
from descartes import PolygonPatch
import fiona
from shapely.geometry import Polygon, MultiPolygon, shape

# We can extract the London Borough boundaries by filtering on the AREA_CODE key
mp = MultiPolygon(
    [shape(pol['geometry']) for pol in fiona.open('data/boroughs/boroughs.shp')
    if pol['properties']['AREA_CODE'] == 'LBO'])

jdickinson202

import scipy
import numpy as np
import pandas as pd
import statsmodels.api as sm
from statsmodels.tools.tools import chain_dot
from statsmodels.tools.decorators import cache_readonly
from statsmodels.regression.linear_model import (RegressionModel,
                                                 RegressionResults)

class Group():

jdickinson202

jdickinson202

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "You can write directly into latex using \"markdown\" by highlighting a cell and then selecting \"markdown\" from the dropdown menu: $$4+x_2^4 = 23$$"
   ]
  },
  {

jdickinson202

import fiona
import os
import rtree

from collections import defaultdict
from shapely.geometry import shape

CONTAINS = 'contains'
CROSSES = 'crosses'
DISJOINT = 'disjoint'

jdickinson202

import sys
import urllib
import urllib2
import json
import logging


class GeonamesError(Exception):
    
    def __init__(self, status):

jdickinson202

import requests
import json


# For information on endpoints and arguments see the geonames
# API documentation at:
# 
#   http://www.geonames.org/export/web-services.html

class Geonames(object):

jdickinson202

#!/usr/bin/env python

from optparse import OptionParser
import csv
import json
import StringIO
import sys
import urllib

# Print the distance between an origin and multiple destinations.