kevin-snippet

#Common Commands

System Utility

• Upgrade all packages with pip

sudo pip freeze --local | grep -v '^\-e' | cut -d = -f 1  | xargs -n1 sudo pip install -U

• Split and Concat files in Shell

split -l 10000 new_data_lower.txt

kevin-snippet

sudo defaults write /Library/Preferences/com.apple.windowserver DisplayResolutionEnabled -bool YES;
sudo defaults delete /Library/Preferences/com.apple.windowserver DisplayResolutionDisabled;

// by the way, you need to logout and log back in for this to take effect.  Or at least that's what
// Quartz Debug says.  Who knows, maybe it's lying?

// P.S. Go to [Apple menu --> System Preferences --> Displays --> Display --> Scaled] after logging
// back in, and you'll see a bunch of "HiDPI" resolutions in the list to choose from.

kevin-snippet

Transforming Code into Beautiful, Idiomatic Python

Notes from Raymond Hettinger's talk at pycon US 2013 video, slides.

Looping over a range of numbers

for i in [0, 1, 2, 3, 4, 5]:
    print i**2

kevin-snippet

    # input list from csv files
    file = open('input_file_name.csv', 'r')
    data = csv.reader(file)
    urls = list(data)
    url = []
    for i in urls:
        url.append(i[0])
    start_urls = url[:]
    file.close()

kevin-snippet

BOT_NAME = 'so_users'

SPIDER_MODULES = ['so_users.spiders']
NEWSPIDER_MODULE = 'so_users.spiders'
DEFAULT_ITEM_CLASS = 'so_users.items.SoUsersItem'

ITEM_PIPELINES = [
#     'goodread.pipelines.RequiredFieldsPipeline',
# #    'dirbot.pipelines.FilterWordsPipeline',
     'so_users.pipelines.MySQLStorePipeline',

kevin-snippet

CREATE TABLE user (
    user_id INT Primary Key,
    userXXXXX
    updated datetime
) DEFAULT CHARSET=utf8;

CREATE TABLE follower (
    user_id INT, -- References user.
    follower_id INT,  -- References user.
    PRIMARY KEY (user_id, follower_id),

kevin-snippet

from scrapy.item import Item, Field
from scrapy.contrib.loader import XPathItemLoader
from scrapy.contrib.loader.processor import TakeFirst


class XXXXX(Item):

    item1 = Field()
    item2 = Field()
    item3 = Field()

kevin-snippet

DROP TABLE IF EXISTS table_name;
CREATE TABLE user (
var1 varchar(155) PRIMARY KEY,
var2 int ,
var3 varchar(155),
var4 varchar(155),
var5 varchar(155),
updated datetime
) DEFAULT CHARSET=utf8;

kevin-snippet

class bcolors:
    HEADER = '\033[95m'
    OKBLUE = '\033[94m'
    OKGREEN = '\033[92m'
    WARNING = '\033[93m'
    FAIL = '\033[91m'
    ENDC = '\033[0m'
    BOLD = '\033[1m'
    UNDERLINE = '\033[4m'