R 101

Sometimes when you open a data file (lets say a .csv), variables will be recognized as factor whereas it should be numeric. It is therefore tempting to simply convert the variable to numeric using as.numeric(). Big mistake! If you use directly as.numeric() on a variable of the type factor, R will returns the levels of the factor rather the actual values. To overcome this hassle, you should first convert the variable into a string with as.character() and thereafter use as.numeric().

x <- as.factor(rnorm(5))
x
##  1.14014396149986   -1.22818036295872  -1.40422561741145
##  0.822237046782029  -0.601888249943712
## 5 Levels: -1.40422561741145 -1.22818036295872 ... 1.14014396149986
## Wrong way
as.numeric(x)
##  5 2 1 4 3
## Right way
as.numeric(as.character(x))
##   1.1401440 -1.2281804 -1.4042256  0.8222370 -0.6018882

Another way to prevent numeric to be converted into factor would be to use as.is = T when opening the data file with read.csv.

Remove all objects in the workspace**

rm(list = ls())

Replacing values of a data frame with NA

library(xtable)

## Random matrix with with some (50) zeros
df <- matrix(rnorm(100), ncol = 10)

df[sample(100, 50)] <- 0

df <- data.frame(df)

print(xtable(df), type = "html")
X1 X2 X3 X4 X5 X6 X7 X8 X9 X10
1 0.00 0.00 0.00 0.00 -0.74 -0.80 0.00 -0.49 0.00 0.00
2 0.00 0.40 0.00 0.00 -0.58 -0.09 0.39 0.00 0.00 -0.14
3 0.00 0.00 -0.14 0.00 0.14 0.00 0.00 2.18 0.00 0.00
4 0.00 0.56 0.00 1.06 0.43 0.55 0.00 -0.76 0.00 0.00
5 0.00 0.00 0.00 0.00 -1.94 -0.64 0.00 0.94 1.90 0.00
6 1.46 0.00 -0.63 -1.14 -0.56 0.44 0.45 -1.25 0.00 0.69
7 0.27 0.00 -0.75 -1.28 0.00 -0.77 0.00 -2.95 0.00 0.00
8 0.00 -0.10 0.31 0.00 0.18 -0.02 0.00 0.00 0.00 0.00
9 0.00 0.50 0.00 -1.43 0.00 1.15 0.11 0.60 0.00 0.00
10 0.69 0.53 0.00 -1.45 -0.72 0.00 0.20 0.94 0.97 1.00
## Replace the 0 with NA
df[df == 0] <- NA

print(xtable(df), type = "html", NA.string = "NA")
X1 X2 X3 X4 X5 X6 X7 X8 X9 X10
1 NA NA NA NA -0.74 -0.80 NA -0.49 NA NA
2 NA 0.40 NA NA -0.58 -0.09 0.39 NA NA -0.14
3 NA NA -0.14 NA 0.14 NA NA 2.18 NA NA
4 NA 0.56 NA 1.06 0.43 0.55 NA -0.76 NA NA
5 NA NA NA NA -1.94 -0.64 NA 0.94 1.90 NA
6 1.46 NA -0.63 -1.14 -0.56 0.44 0.45 -1.25 NA 0.69
7 0.27 NA -0.75 -1.28 NA -0.77 NA -2.95 NA NA
8 NA -0.10 0.31 NA 0.18 -0.02 NA NA NA NA
9 NA 0.50 NA -1.43 NA 1.15 0.11 0.60 NA NA
10 0.69 0.53 NA -1.45 -0.72 NA 0.20 0.94 0.97 1.00

##Count % of missing values per column

The other day I was working with a large database. I was looking for a way to calculate % of missing values for each variable. Here is how I calculated it.

## Generate some data
df <- matrix(rnorm(100), ncol = 10)

df[sample(100, 50)] <- NA

df <- data.frame(df)

print(xtable(df), type = "html", NA.string = "NA")
X1 X2 X3 X4 X5 X6 X7 X8 X9 X10
1 -0.72 -2.19 NA NA 0.85 -1.71 NA NA NA NA
2 -0.19 NA 0.77 2.82 NA 0.18 NA NA NA NA
3 NA NA -1.01 -0.90 NA 0.66 NA 0.30 NA 1.05
4 NA -1.96 NA NA 0.59 NA -0.07 1.33 NA NA
5 -0.00 NA 0.58 -1.77 0.79 -1.29 NA 0.39 NA -1.28
6 -0.60 NA NA 0.34 -0.94 -0.35 NA NA 0.80 -0.68
7 NA -0.23 -1.27 NA -0.85 -0.60 NA NA NA -0.34
8 -0.07 NA NA NA 0.60 NA NA 0.91 -0.40 -1.37
9 0.23 NA -0.95 NA 1.66 0.43 0.12 NA NA NA
10 0.74 NA -0.53 NA 0.59 NA -0.60 NA 0.90 NA
## Find % of missing values in each column with apply
missing <- apply(df, 2, function(x) sum(is.na(x)))/nrow(df)*100

## Print the result
df2 <- data.frame(variable = names(df), missing = missing)

print(xtable(df2), type = "html", include.rownames = FALSE)
variable missing
X1 30.00
X2 70.00
X3 40.00
X4 60.00
X5 20.00
X6 30.00
X7 70.00
X8 60.00
X9 70.00
X10 50.00

Grouping data frame

Tsshere is various way to merge data frame in R. For me, the most easiest way is to use the join functions from the dplyr package. library(dplyr)
##
## Attaching package: 'dplyr'
## The following objects are masked from 'package:stats':
##
##     filter, lag
## The following objects are masked from 'package:base':
##
##     intersect, setdiff, setequal, union
df1 <- data.frame(ID = sample(LETTERS[1:20],10, replace = FALSE), x = rnorm(10))
df2 <- data.frame(ID = sample(LETTERS[1:20],10, replace = FALSE), y = rnorm(10))

print(xtable(df1), type = "html", include.rownames = FALSE)
ID x
D -1.71
Q -0.34
H -0.24
A -0.58
E -0.71
C 0.15
K -0.31
O -0.93
J -1.26
S 0.17
print(xtable(df2), type = "html", include.rownames = FALSE)
ID y
T -1.11
B -0.78
R 0.37
E 0.04
D -1.02
G -1.72
P 1.16
K 1.02
S -0.13
J 1.06
inner_join(df1, df2, by = "ID")
## Warning: Column ID joining factors with different levels, coercing to
## character vector

ID x y 1 D -1.7139254 -1.02036670 2 E -0.7134614 0.03531296 3 K -0.3101741 1.02019141 4 J -1.2575589 1.05794772 5 S 0.1670762 -0.12569667

df_exact_match <- merge(df1,df2, by = "ID") ## Exact match
print(xtable(df_exact_match), type = "html", include.rownames = FALSE, NA.string = "NA")
ID x y
D -1.71 -1.02
E -0.71 0.04
J -1.26 1.06
K -0.31 1.02
S 0.17 -0.13
df_all_x <- merge(df1,df2, by = "ID", all.x = T) ## Keep all x (discard y that are not matched)
print(xtable(df_all_x), type = "html", include.rownames = FALSE, NA.string = "NA")
ID x y
A -0.58 NA
C 0.15 NA
D -1.71 -1.02
E -0.71 0.04
H -0.24 NA
J -1.26 1.06
K -0.31 1.02
O -0.93 NA
Q -0.34 NA
S 0.17 -0.13
df_all_y <- merge(df1,df2, by = "ID", all.y = T) ## Keep all y (discard x that are not matched)
print(xtable(df_all_y), type = "html", include.rownames = FALSE, NA.string = "NA")
ID x y
D -1.71 -1.02
E -0.71 0.04
J -1.26 1.06
K -0.31 1.02
S 0.17 -0.13
B NA -0.78
G NA -1.72
P NA 1.16
R NA 0.37
T NA -1.11
df_all <- merge(df1,df2, by = "ID", all = T) ## Keep everything
print(xtable(df_all), type = "html", include.rownames = FALSE, NA.string = "NA")
ID x y
A -0.58 NA
C 0.15 NA
D -1.71 -1.02
E -0.71 0.04
H -0.24 NA
J -1.26 1.06
K -0.31 1.02
O -0.93 NA
Q -0.34 NA
S 0.17 -0.13
B NA -0.78
G NA -1.72
P NA 1.16
R NA 0.37
T NA -1.11