Contents

$$\newcommand{\vect}[1]{\boldsymbol{#1}}$$ $$\newcommand{\transp}{^{\text{T}}}$$ $$\newcommand{\mat}[1]{\boldsymbol{\mathcal{#1}}}$$ $$\newcommand{\sign}{\text{sign}}$$

1 Approximation Ability

1.1 Example: $$f(x)=x^2$$

set.seed(1)
x <- runif(50,-1,1)
x <- sort(x)
y <- x**2

library(keras)
set.seed(1)
model <- keras_model_sequential()
model %>%
layer_dense(units = 3, activation = "tanh", input_shape = c(1)) %>%
layer_dense(units = 1, activation = "linear") %>%
compile(
optimizer = optimizer_sgd(),
loss = "mean_squared_error",
metrics = c("mse")
)

summary(model)
Model: "sequential"
________________________________________________________________________________
Layer (type)                        Output Shape                    Param #
================================================================================
dense_1 (Dense)                     (None, 3)                       6
________________________________________________________________________________
dense (Dense)                       (None, 1)                       4
================================================================================
Total params: 10
Trainable params: 10
Non-trainable params: 0
________________________________________________________________________________
history <- model %>% fit(x, y, epochs = 30000, batch_size = 50,
validation_split = 0,verbose=0)
plot(history)
geom_smooth() using formula 'y ~ x'

model %>% evaluate(x, y)
        loss          mse
0.0007400098 0.0007400098 
y_pred = model %>% predict(x)
plot(y~x,col="blue")
points(x,y_pred,col="red",type="l")
legend("top", legend=c( "y-predicted","y-original"),
col=c("red", "blue"), lty=c(1,NA),cex=0.8,pch=c(NA,1))

1.2 Absolute example

set.seed(1)
x <- runif(50,-1,1)
x <- sort(x)
y <- abs(x)

library(keras)
set.seed(1)
model1 <- keras_model_sequential()
model1 %>%
layer_dense(units = 3, activation = "tanh", input_shape = c(1)) %>%
layer_dense(units = 1, activation = "linear") %>%
compile(
optimizer = optimizer_sgd(),
loss = "mean_squared_error",
metrics = c("mse")
)

summary(model1)
Model: "sequential_1"
________________________________________________________________________________
Layer (type)                        Output Shape                    Param #
================================================================================
dense_3 (Dense)                     (None, 3)                       6
________________________________________________________________________________
dense_2 (Dense)                     (None, 1)                       4
================================================================================
Total params: 10
Trainable params: 10
Non-trainable params: 0
________________________________________________________________________________
history <- model1 %>% fit(x, y, epochs = 30000, batch_size = 50,
validation_split = 0,verbose=0)
plot(history)
geom_smooth() using formula 'y ~ x'

model1 %>% evaluate(x, y)
        loss          mse
0.0009216355 0.0009216355 
y_pred = model1 %>% predict(x)
plot(y~x,col="blue")
points(x,y_pred,col="red",type="l")
legend("top", legend=c( "y-predicted","y-original"),
col=c("red", "blue"), lty=c(1,NA),cex=0.8,pch=c(NA,1))

1.3 Example: $$f(x)=sin(x)$$

set.seed(1)
x <- runif(50,-1,1)
x <- sort(x)
y <- sin(x*pi)

library(keras)
set.seed(1)
model2 <- keras_model_sequential()
model2 %>%
layer_dense(units = 3, activation = "tanh", input_shape = c(1)) %>%
layer_dense(units = 1, activation = "linear") %>%
compile(
optimizer = optimizer_sgd(),
loss = "mean_squared_error",
metrics = c("mse")
)

summary(model2)
Model: "sequential_2"
________________________________________________________________________________
Layer (type)                        Output Shape                    Param #
================================================================================
dense_5 (Dense)                     (None, 3)                       6
________________________________________________________________________________
dense_4 (Dense)                     (None, 1)                       4
================================================================================
Total params: 10
Trainable params: 10
Non-trainable params: 0
________________________________________________________________________________
history <- model2 %>% fit(x, y, epochs = 30000, batch_size = 50,
validation_split = 0,verbose=0)
plot(history)
geom_smooth() using formula 'y ~ x'