#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
This module contains procedures provide in the "Udacity Self-Driving Car Engineer
Nanodegree" lecture notes. Some modifications were made.
https://www.udacity.com/course/self-driving-car-engineer-nanodegree--nd013
"""
import matplotlib.image as mpimg
import numpy as np
import cv2
import time
from skimage.feature import hog
from sklearn.svm import LinearSVC
from sklearn.preprocessing import StandardScaler
# NOTE: the next import is only valid for scikit-learn version <= 0.17
# for scikit-learn >= 0.18 use:
# from sklearn.model_selection import train_test_split
from sklearn.cross_validation import train_test_split
from sklearn.metrics import accuracy_score
from scipy.ndimage.measurements import label
[docs]class Params():
def __init__(self):
self.windows = []
self.hot_windows = []
[docs]class Log():
def __init__(self):
self.ss = Params()
self.nss = Params()
log = Log()
[docs]def convert_color(img, conv='RGB2YCrCb'):
if conv == 'RGB2YCrCb':
return cv2.cvtColor(img, cv2.COLOR_RGB2YCrCb)
if conv == 'BGR2YCrCb':
return cv2.cvtColor(img, cv2.COLOR_BGR2YCrCb)
if conv == 'RGB2LUV':
return cv2.cvtColor(img, cv2.COLOR_RGB2LUV)
[docs]def get_hog_features(img, orient, pix_per_cell, cell_per_block,
vis=False, feature_vec=True):
"""Define a function to return HOG features and visualization"""
# Call with two outputs if vis==True
if vis == True:
features, hog_image = hog(img, orientations=orient,
pixels_per_cell=(pix_per_cell, pix_per_cell),
cells_per_block=(cell_per_block, cell_per_block),
transform_sqrt=True,
visualise=vis, feature_vector=feature_vec)
return features, hog_image
# Otherwise call with one output
else:
features = hog(img, orientations=orient,
pixels_per_cell=(pix_per_cell, pix_per_cell),
cells_per_block=(cell_per_block, cell_per_block),
transform_sqrt=True,
visualise=vis, feature_vector=feature_vec)
return features
[docs]def bin_spatial(img, size=(32, 32)):
"""Define a function to compute binned color features"""
color1 = cv2.resize(img[:,:,0], size).ravel()
color2 = cv2.resize(img[:,:,1], size).ravel()
color3 = cv2.resize(img[:,:,2], size).ravel()
return np.hstack((color1, color2, color3))
# old implementation on next two lines below
features = cv2.resize(img, size).ravel() # Use cv2.resize().ravel() to create the feature vector
return features # Return the feature vector
[docs]def color_hist(img, nbins=32, bins_range=(0, 256)):
"""Define a function to compute color histogram features"""
# NEED TO CHANGE bins_range if reading .png files with mpimg!
# Compute the histogram of the color channels separately
channel1_hist = np.histogram(img[:,:,0], bins=nbins, range=bins_range)
channel2_hist = np.histogram(img[:,:,1], bins=nbins, range=bins_range)
channel3_hist = np.histogram(img[:,:,2], bins=nbins, range=bins_range)
# Concatenate the histograms into a single feature vector
hist_features = np.concatenate((channel1_hist[0], channel2_hist[0], channel3_hist[0]))
# Return the individual histograms, bin_centers and feature vector
return hist_features
[docs]def colorspace(image, color_space):
if color_space != 'RGB':
if color_space == 'HSV':
feature_image = cv2.cvtColor(image, cv2.COLOR_RGB2HSV)
elif color_space == 'LUV':
feature_image = cv2.cvtColor(image, cv2.COLOR_RGB2LUV)
elif color_space == 'HLS':
feature_image = cv2.cvtColor(image, cv2.COLOR_RGB2HLS)
elif color_space == 'YUV':
feature_image = cv2.cvtColor(image, cv2.COLOR_RGB2YUV)
elif color_space == 'YCrCb':
feature_image = cv2.cvtColor(image, cv2.COLOR_RGB2YCrCb)
else:
raise ValueError('Unknown color space', color_space)
else: feature_image = np.copy(image)
if np.max(feature_image) > 2: # assume it is 0-255 and not 0-1
feature_image = feature_image.astype(np.float32)/255
return feature_image
[docs]def single_img_features(img, color_space='RGB', spatial_size=(32, 32),
hist_bins=32, orient=9,
pix_per_cell=8, cell_per_block=2, channel=0,
spatial_feat=True, hist_feat=True, hog_feat=True):
"""
Define a function to extract features from a single image window
This function is very similar to extract_features()
just for a single image rather than list of images
"""
#1) Define an empty list to receive features
img_features = []
#2) Apply color conversion if other than 'RGB'
feature_image = colorspace(img, color_space)
#3) Compute spatial features if flag is set
if spatial_feat == True:
spatial_features = bin_spatial(feature_image, size=spatial_size)
#4) Append features to list
img_features.append(spatial_features)
#5) Compute histogram features if flag is set
if hist_feat == True:
hist_features = color_hist(feature_image, nbins=hist_bins)
#6) Append features to list
img_features.append(hist_features)
#7) Compute HOG features if flag is set
if hog_feat == True:
if channel == 'ALL' or channel > 2:
hog_features = []
for ch in range(feature_image.shape[2]):
hog_features.extend(get_hog_features(feature_image[:,:,ch],
orient, pix_per_cell, cell_per_block,
vis=False, feature_vec=True))
else:
hog_features = get_hog_features(feature_image[:,:,channel], orient,
pix_per_cell, cell_per_block, vis=False, feature_vec=True)
#8) Append features to list
img_features.append(hog_features)
#9) Return concatenated array of features
return np.concatenate(img_features)
[docs]def train_svm(cars,
notcars,
color_space = 'RGB', # Can be RGB, HSV, LUV, HLS, YUV, YCrCb
orient = 9, # HOG orientations
pix_per_cell = 8, # HOG pixels per cell
cell_per_block = 2, # HOG cells per block
channel = 0, # Can be 0, 1, 2, or "ALL"
spatial_size = (16, 16), # Spatial binning dimensions
hist_bins = 16, # Number of histogram bins
spatial_feat = True, # Spatial features on or off
hist_feat = True, # Histogram features on or off
hog_feat = True): # HOG features on or off
t1 = time.time()
car_features = extract_features(cars, color_space=color_space,
spatial_size=spatial_size, hist_bins=hist_bins,
orient=orient, pix_per_cell=pix_per_cell,
cell_per_block=cell_per_block,
channel=channel, spatial_feat=spatial_feat,
hist_feat=hist_feat, hog_feat=hog_feat)
notcar_features = extract_features(notcars, color_space=color_space,
spatial_size=spatial_size, hist_bins=hist_bins,
orient=orient, pix_per_cell=pix_per_cell,
cell_per_block=cell_per_block,
channel=channel, spatial_feat=spatial_feat,
hist_feat=hist_feat, hog_feat=hog_feat)
t2 = time.time()
X = np.vstack((car_features, notcar_features)).astype(np.float64)
# Fit a per-column scaler
X_scaler = StandardScaler().fit(X)
# Apply the scaler to X
scaled_X = X_scaler.transform(X)
# Define the labels vector
y = np.hstack((np.ones(len(car_features)), np.zeros(len(notcar_features))))
# Split up data into randomized training and test sets
rand_state = np.random.randint(0, 100)
X_train, X_test, y_train, y_test = train_test_split(
scaled_X, y, test_size=0.2, random_state=rand_state)
# Use a linear SVC
svc = LinearSVC()
# Check the training time for the SVC
t3=time.time()
svc.fit(X_train, y_train)
t4 = time.time()
prediction = svc.predict(X_test)
cfg_str = '{} in training set with feature vector length of {}'
print(cfg_str.format(len(X_train), len(X_train[0])))
ftr_str = 'Features extracted in {} seconds and trained SVC in {} with accuracy {}.'
print(ftr_str.format(round(t2-t1, 0), round(t4-t3, 0), round(accuracy_score(prediction, y_test),2)))
stu_str = 'Using {} orientations {} pixels per cell and {} cells per block'
print(stu_str.format(orient, pix_per_cell, cell_per_block,))
return svc, X_scaler
[docs]def add_heat(heatmap, bbox_list):
"""Iterate through list of bboxes"""
for box in bbox_list:
# Add += 1 for all pixels inside each bbox
# Assuming each "box" takes the form ((x1, y1), (x2, y2))
heatmap[box[0][1]:box[1][1], box[0][0]:box[1][0]] += 1
# Return updated heatmap
return heatmap# Iterate through list of bboxes
[docs]def apply_threshold(heatmap, threshold):
"""Zero out pixels below the threshold"""
heatmap[heatmap <= threshold] = 0
# Return thresholded map
return heatmap
[docs]def draw_labeled_bboxes(img, labels):
"""Iterate through all detected cars"""
for car_number in range(1, labels[1]+1):
# Find pixels with each car_number label value
nonzero = (labels[0] == car_number).nonzero()
# Identify x and y values of those pixels
nonzeroy = np.array(nonzero[0])
nonzerox = np.array(nonzero[1])
# Define a bounding box based on min/max x and y
bbox = ((np.min(nonzerox), np.min(nonzeroy)), (np.max(nonzerox), np.max(nonzeroy)))
# Draw the box on the image
cv2.rectangle(img, bbox[0], bbox[1], (255,255,255), 6)
# Return the image
return img
[docs]def heat_map(image, box_list, threshold=1):
"""Create a heat map based box list and threshold"""
heat = np.zeros_like(image[:,:,0]).astype(np.float)
# Add heat to each box in box list
heat = add_heat(heat,box_list)
# Apply threshold to help remove false positives
heat = apply_threshold(heat,threshold)
# Visualize the heatmap when displaying
heatmap = np.clip(heat, 0, 255)
# Find final boxes from heatmap using label function
labels = label(heatmap)
return heatmap, labels
[docs]def draw_boxes(img, bboxes, color=(0, 0, 255), thick=6):
"""Define a function to draw bounding boxes"""
# Make a copy of the image
imcopy = np.copy(img)
# Iterate through the bounding boxes
for bbox in bboxes:
# Draw a rectangle given bbox coordinates
cv2.rectangle(imcopy, bbox[0], bbox[1], color, thick)
# Return the image copy with boxes drawn
return imcopy
# Define a single function that can extract features using hog sub-sampling and make predictions
[docs]def find_cars(img, svc, X_scaler, ystart=400, ystop=650, scale=1, spatial_size=(32,32), hist_bins=32,
orient=9, pix_per_cell=8, cell_per_block=2):
global log
# draw_img = np.copy(img) # uncomment if you want to return an image
img = img.astype(np.float32)/255
img_tosearch = img[ystart:ystop,:,:]
ctrans_tosearch = convert_color(img_tosearch, conv='RGB2YCrCb')
if scale != 1:
imshape = ctrans_tosearch.shape
ctrans_tosearch = cv2.resize(ctrans_tosearch, (np.int(imshape[1]/scale), np.int(imshape[0]/scale)))
ch1 = ctrans_tosearch[:,:,0]
ch2 = ctrans_tosearch[:,:,1]
ch3 = ctrans_tosearch[:,:,2]
# Compute individual channel HOG features for the entire image
hog1 = get_hog_features(ch1, orient, pix_per_cell, cell_per_block, feature_vec=False)
hog2 = get_hog_features(ch2, orient, pix_per_cell, cell_per_block, feature_vec=False)
hog3 = get_hog_features(ch3, orient, pix_per_cell, cell_per_block, feature_vec=False)
# Define blocks and steps as above
nxblocks = (ch1.shape[1] // pix_per_cell)-1
nyblocks = (ch1.shape[0] // pix_per_cell)-1
window_list = []
#nfeat_per_block = orient*cell_per_block**2
# 64 was the orginal sampling rate, with 8 cells and 8 pix per cell
window = 64
nblocks_per_window = (window // pix_per_cell)-1
cells_per_step = 2 # Instead of overlap, define how many cells to step
nxsteps = (nxblocks - nblocks_per_window) // cells_per_step
nysteps = (nyblocks - nblocks_per_window) // cells_per_step
for xb in range(nxsteps):
for yb in range(nysteps):
ypos = yb*cells_per_step
xpos = xb*cells_per_step
# Extract HOG for this patch
hog_feat1 = hog1[ypos:ypos+nblocks_per_window, xpos:xpos+nblocks_per_window].ravel()
hog_feat2 = hog2[ypos:ypos+nblocks_per_window, xpos:xpos+nblocks_per_window].ravel()
hog_feat3 = hog3[ypos:ypos+nblocks_per_window, xpos:xpos+nblocks_per_window].ravel()
hog_features = np.hstack((hog_feat1, hog_feat2, hog_feat3))
xleft = xpos*pix_per_cell
ytop = ypos*pix_per_cell
# Extract the image patch
subimg = cv2.resize(ctrans_tosearch[ytop:ytop+window, xleft:xleft+window], (64,64))
# Get color features
spatial_features = bin_spatial(subimg, size=spatial_size)
hist_features = color_hist(subimg, nbins=hist_bins)
# Scale features and make a prediction
test_features = X_scaler.transform(np.hstack((spatial_features, hist_features, hog_features)).reshape(1, -1))
#test_features = X_scaler.transform(np.hstack((shape_feat, hist_feat)).reshape(1, -1))
test_prediction = svc.predict(test_features)
xbox_left = np.int(xleft*scale)
ytop_draw = np.int(ytop*scale)
win_draw = np.int(window*scale)
log.ss.windows.append(((xbox_left, ytop_draw+ystart), (xbox_left+win_draw, ytop_draw+win_draw+ystart)))
if test_prediction == 1:
window_list.append(((xbox_left, ytop_draw+ystart), (xbox_left+win_draw, ytop_draw+win_draw+ystart)))
#cv2.rectangle(draw_img,(xbox_left, ytop_draw+ystart),(xbox_left+win_draw,ytop_draw+win_draw+ystart),(0,0,255),6)
log.ss.hot_windows.append(((xbox_left, ytop_draw+ystart), (xbox_left+win_draw, ytop_draw+win_draw+ystart)))
return window_list # draw_img # old return value commented out
[docs]def slide_window(img, x_start_stop=[None, None], y_start_stop=[None, None],
xy_window=(64, 64), xy_overlap=(0.75, 0.75)):
"""
Define a function that takes an image,
start and stop positions in both x and y,
window size (x and y dimensions),
and overlap fraction (for both x and y)
"""
global log
# If x and/or y start/stop positions not defined, set to image size
if x_start_stop[0] == None:
x_start_stop[0] = 0
if x_start_stop[1] == None:
x_start_stop[1] = img.shape[1]
if y_start_stop[0] == None:
y_start_stop[0] = 0
if y_start_stop[1] == None:
y_start_stop[1] = img.shape[0]
# Compute the span of the region to be searched
xspan = x_start_stop[1] - x_start_stop[0]
yspan = y_start_stop[1] - y_start_stop[0]
# Compute the number of pixels per step in x/y
nx_pix_per_step = np.int(xy_window[0]*(1 - xy_overlap[0]))
ny_pix_per_step = np.int(xy_window[1]*(1 - xy_overlap[1]))
# Compute the number of windows in x/y
nx_buffer = np.int(xy_window[0]*(xy_overlap[0]))
ny_buffer = np.int(xy_window[1]*(xy_overlap[1]))
nx_windows = np.int((xspan-nx_buffer)/nx_pix_per_step)
ny_windows = np.int((yspan-ny_buffer)/ny_pix_per_step)
# Initialize a list to append window positions to
window_list = []
# Loop through finding x and y window positions
# Note: you could vectorize this step, but in practice
# you'll be considering windows one by one with your
# classifier, so looping makes sense
for xs in range(nx_windows):
for ys in range(ny_windows):
# Calculate window position
startx = xs*nx_pix_per_step + x_start_stop[0]
endx = startx + xy_window[0]
starty = ys*ny_pix_per_step + y_start_stop[0]
endy = starty + xy_window[1]
# Append window position to list
window_list.append(((startx, starty), (endx, endy)))
log.nss.windows.append(((startx, starty), (endx, endy)))
# Return the list of windows
return window_list
[docs]def detect_cars_in_image(image,
svc,
X_scaler,
color_space = 'RGB', # Can be RGB, HSV, LUV, HLS, YUV, YCrCb
orient = 9, # HOG orientations
pix_per_cell = 8, # HOG pixels per cell
cell_per_block = 2, # HOG cells per block
channel = 0, # Can be 0, 1, 2, or "ALL"
spatial_size = (16, 16), # Spatial binning dimensions
hist_bins = 16, # Number of histogram bins
spatial_feat = True, # Spatial features on or off
hist_feat = True, # Histogram features on or off
hog_feat = True, # HOG features on or off
y_start_stop = [400, 650],
xy_windows = [(64, 64)]): # Min and max in y to search in slide_window()
# Uncomment the following line if you extracted training
# data from .png images (scaled 0 to 1 by mpimg) and the
# image you are searching is a .jpg (scaled 0 to 255)
if np.max(image) > 2: # assume jpg
image = image.astype(np.float32)/255
xy_windows = [(100, 100),(150,150)]
xy_overlap=(0.75, 0.75)
#xy_overlap=(0.8, 0.8)
windows = []
for xy_window in xy_windows:
windows.extend(slide_window(image, x_start_stop=[None, None], y_start_stop=y_start_stop,
xy_window=xy_window, xy_overlap=xy_overlap))
hot_windows = search_windows(image, windows, svc, X_scaler, color_space=color_space,
spatial_size=spatial_size, hist_bins=hist_bins,
orient=orient, pix_per_cell=pix_per_cell,
cell_per_block=cell_per_block,
channel=channel, spatial_feat=spatial_feat,
hist_feat=hist_feat, hog_feat=hog_feat)
log.nss.hot_windows.extend(hot_windows)
return hot_windows
[docs]def search_windows(img, windows, clf, scaler, color_space='RGB',
spatial_size=(32, 32), hist_bins=32,
hist_range=(0, 256), orient=9,
pix_per_cell=8, cell_per_block=2,
channel=0, spatial_feat=True,
hist_feat=True, hog_feat=True):
"""
Define a function you will pass an image
and the list of windows to be searched (output of slide_windows())
"""
#1) Create an empty list to receive positive detection windows
on_windows = []
#2) Iterate over all windows in the list
for window in windows:
#3) Extract the test window from original image
test_img = cv2.resize(img[window[0][1]:window[1][1], window[0][0]:window[1][0]], (64, 64))
#4) Extract features for that window using single_img_features()
features = single_img_features(test_img, color_space=color_space,
spatial_size=spatial_size, hist_bins=hist_bins,
orient=orient, pix_per_cell=pix_per_cell,
cell_per_block=cell_per_block,
channel=channel, spatial_feat=spatial_feat,
hist_feat=hist_feat, hog_feat=hog_feat)
#5) Scale extracted features to be fed to classifier
test_features = scaler.transform(np.array(features).reshape(1, -1))
#6) Predict using your classifier
prediction = clf.predict(test_features)
#7) If positive (prediction == 1) then save the window
if prediction == 1:
on_windows.append(window)
#8) Return windows for positive detections
return on_windows