1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
use std::fmt;
use std::io;
use crate::tensor::*;
pub(crate) use self::batch_iterator::BatchIterator;
pub use self::image_data::ImageDataSet;
pub use self::image_data::ImageDataSetBuilder;
pub use self::image_data::ImageOps;
pub use self::tabular_data::TabularDataSet;
mod batch_iterator;
mod image_data;
mod tabular_data;
#[derive(Debug)]
pub enum DataSetError {
Io(io::Error),
Csv(csv::Error),
DimensionMismatch,
PathDoesNotExist,
TrainPathDoesNotExist,
ValidPathDoesNotExist,
ImageFormatNotSupported,
InvalidImagePath,
InvalidValidationFraction,
DifferentNumbersOfChannels,
}
#[derive(Debug)]
enum IO {
Input,
Output,
}
impl fmt::Display for DataSetError {
fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
match *self {
DataSetError::Io(ref err) => write!(f, "IO error: {}", err),
DataSetError::Csv(ref err) => write!(f, "CSV error: {}", err),
DataSetError::DimensionMismatch => write!(f, "The number of input and output samples differ."),
DataSetError::PathDoesNotExist => write!(f, "The path does not exist."),
DataSetError::TrainPathDoesNotExist => write!(f, "The root directory does not contain a 'train' subfolder."),
DataSetError::ValidPathDoesNotExist => write!(f, "The root directory does not contain a 'valid' subfolder."),
DataSetError::ImageFormatNotSupported => write!(f, "The image format is not supported."),
DataSetError::InvalidImagePath => write!(f, "The path could not be opened as an image."),
DataSetError::InvalidValidationFraction => write!(f, "The validation fraction is incorrect. It must be between 0 and 1."),
DataSetError::DifferentNumbersOfChannels => write!(f, "The directory contains images with different numbers of channels."),
}
}
}
impl std::convert::From<io::Error> for DataSetError {
fn from(error: io::Error) -> DataSetError {
DataSetError::Io(error)
}
}
#[derive(Debug)]
pub enum Scaling {
Normalized,
Standarized,
}
pub trait DataSet {
fn input_shape(&self) -> Dim;
fn output_shape(&self) -> Dim;
fn num_train_samples(&self) -> u64;
fn num_valid_samples(&self) -> u64;
fn classes(&self) -> Option<Vec<String>> { None }
fn x_train(&self) -> &Tensor;
fn y_train(&self) -> &Tensor;
fn x_valid(&self) -> Option<&Tensor>;
fn y_valid(&self) -> Option<&Tensor>;
fn x_test(&self) -> Option<&Tensor>;
fn y_test(&self) -> Option<&Tensor>;
fn x_train_stats(&self) -> &Option<(Scaling, Tensor, Tensor)>;
fn y_train_stats(&self) -> &Option<(Scaling, Tensor, Tensor)>;
}