Assignment 2: Group 23 - Githubissues

title: "Assignment 2" author: "Group 23" date: "8. nov. 2015"

output: html_document

# Loading packages
library("rvest")
library("stringr")
library("plyr")
library("dplyr")
library("ggplot2")

#################################
# Scraping
#################################

# First we create a list of pages to be scraped
link=list()
for (i in 1:100){
  link[i]<-paste("http://www.ipaidabribe.com/reports/paid?page=",(i-1)*10,"#gsc.tab=0",sep="")
}

# Creating a css

css.selector.title=".heading-3 a"
css.selector.amount=".paid-amount span"
css.selector.namedep=".name a"
css.selector.detail=".transaction a"
css.selector.views=".overview .views"
css.selector.city=".location"
css.selector.date=".date"

bribe<- function(link){
  liste=read_html(link)

  link.title=liste %>%  
    html_nodes(css=css.selector.title) %>%
    html_text() 

  link.amount=liste %>% 
    html_nodes(css=css.selector.amount) %>% 
    html_text()

  link.namedep=liste %>% 
    html_nodes(css=css.selector.namedep) %>% 
    html_text()

  link.detail=liste %>% 
    html_nodes(css=css.selector.detail) %>% 
    html_text()

  link.views=liste %>% 
    html_nodes(css=css.selector.views) %>% 
    html_text()

  link.city=liste %>% 
    html_nodes(css=css.selector.city) %>% 
    html_text()

  link.date=liste %>% 
    html_nodes(css=css.selector.date) %>% 
    html_text()

  return(cbind(link.title,link.amount,link.namedep,link.detail,link.views,link.city,link.date))  
}

heading<- list()
for( i in link[1:100]){
  #print(paste("Processing ",i,sep=""))
  heading[[i]] <- bribe(i)
  #Sys.sleep(1)
  #cat("done !\n")
}
# Making a dataframe
df.head <- ldply(heading)

# Saving dataframe
save(df.head,file="data.Rda")

#############################
# Data manipulation
#############################

# Getting amount paid in numbers
df.head$link.amount <- gsub("Paid INR ","",df.head$link.amount)
df.head$link.amount <- as.numeric(gsub(",","",df.head$link.amount))

# Getting views in numbers
df.head$link.views <- as.numeric(gsub(" views","",df.head$link.views))

#Date
df.head$link.date <- as.Date(df.head$link.date, format="%B %d, %Y")

#Region
df.head$region <- str_extract(df.head$link.city, ", [A-z]+")
df.head$region <- str_extract(df.head$region, "[A-z]+")

# City
df.head$city <- str_extract(df.head$link.city, "[A-z]+")

# Deleting old city variable
df.head$link.city = NULL

names <-rbind("link","title","amount","department","detail","views","date","region","city")
names(df.head) <- names

###############################
# Wikitables
###############################

# Scraping Wiki for number of people in cities
df.india = read_html("https://es.wikipedia.org/wiki/Anexo:Ciudades_de_la_India_por_población") %>%
  html_node(".wikitable") %>% # extract first node with class wikitable
  html_table()
# then convert the HTML table into a data frame

# Keeping colum 2,3 and 5 and making new dataframe
keeps = c(2,3)
df.india.new = df.india[keeps]

# Renaming colums
names = rbind("city", "population")
names(df.india.new) = names

# Renaming cities to match the dataset
df.india.new$city <- gsub("Bombay","Mumbai",df.india.new$city)
df.india.new$city <- gsub("Delhi","New Delhi",df.india.new$city)

#################################
# Merging datasets
#################################

df.merge <- join(df.head,df.india.new,type="left",match="first",by="city")
df.merge$region = NULL
df.merge <- df.merge %>% filter(!is.na(population))

#################################
# Data manipulation
#################################
df.merge$population <- gsub(" ","",df.merge$population)

df.merge$population <- as.numeric(str_extract(df.merge$population,"[0-9]+"))

df.bribe <- df.merge %>% arrange(population)

df.corrupt <- df.bribe %>% 
  group_by(city) %>%
  summarise(bribes=n(),amount=sum(amount),population=mean(population)) %>% 
  mutate(bribe.capita=bribes/population,amount.capita=amount/population,mean.bribe=amount/bribes)

df.corrupt.filter <- df.corrupt

# From now on we only consider cities with more than 14 observations/bribes
# this gives a total of 7 cities

#df.corrupt.filter<- df.corrupt %>% 
 # filter(bribes>14)

Korruption er et problem i Indien, der på trods af bestræbelser på reduktion er stigende. Årsagerne til korruption i Indien omfatter overdrevne regler, komplicerede skatter og licens ordninger, mange ministerier med opak bureaukrati, manglen på gennemsigtige love og processer, magt mm. Som det fremgår af figur 1, er der betydelige forskelle i niveauet af korruption i de forskellige byer. Vi ser at der er relativ flere bestikkelse i Indore, Bangalore og Agra sammenlignet med Mumbai, Hyderabad, Chennai og Pune. Hvilket kan skyldes evnen til og interessen i at reducere korruption i de forskellige byer.

# This is a plot of the number of bribes pr. capita
p_1<- ggplot(data=df.corrupt.filter, aes(x=city, y=bribe.capita))
p_1 <- p_1 + geom_bar(stat="identity")
p_1 <- p_1 + scale_y_continuous("Number of bribes pr. capita")
p_1 <- p_1 + theme_minimal()+ggtitle("Figure 1: Number of bribes pr. capita")
p_1

# This is a plot of the total value of bribes pr. capita

p_2 <- ggplot(data=df.corrupt.filter, aes(x=city, y=amount.capita))
p_2 <- p_2 + geom_bar(stat="identity")
p_2 <- p_2 + scale_y_continuous("Amount of bribes pr. capita")
p_2 <- p_2 + theme_minimal()+ggtitle("Figure 2: Amount of bribes pr. capita")
p_2

I figur 2 ser vi et helt andet billede af bestikkelse i Indien. På trods af antallet af bestikkeler i Mumbai er relativt få, er der tale om en relativ højere værdi. Med andre ord har vi, hvis vi ser bort fra usikkerheds momentet ved at anvende selvraporteret data, fra en hjemmeside med anonymitet, at der i Mumbai er relativ få bestikkelser af stor værdi, hvorimod der i de andre byer er tale om relativ mange bestikkelser af mindre værdi eller relativ for bestikkelser af mindre værdi. Dette kan betyde mange ting, som vi ikke vil komme ind på her.

sebastianbarfort / sds

Assignment 2: Group 23 #47

output: html_document