SageMakerを使ってイメージ分類を、ラベル付け・訓練・推論まで一通り実行する

# 画像をダウンロードして展開
%%bash
wget https://www.cs.toronto.edu/~kriz/cifar-10-python.tar.gz
tar -xvf cifar-10-python.tar.gz
ls cifar-10-batches-py/

---
cifar-10-batches-py/
cifar-10-batches-py/data_batch_4
cifar-10-batches-py/readme.html
cifar-10-batches-py/test_batch
cifar-10-batches-py/data_batch_3
cifar-10-batches-py/batches.meta
cifar-10-batches-py/data_batch_2
cifar-10-batches-py/data_batch_5
cifar-10-batches-py/data_batch_1

import glob

def unpickle(file):
    import pickle
    with open(file, 'rb') as fo:
        dict = pickle.load(fo, encoding='latin-1')
    return dict

def output_data_to_png(input_data, root_dir):
    for i in range(len(input_data['filenames'])):
        filename = input_data['filenames'][i]
        data = input_data['data'][i]
        data = data.reshape(3, 32, 32)
        data = np.swapaxes(data, 0, 2)
        data = np.swapaxes(data, 0, 1)
        label_index = input_data['labels'][i]
        label_name = label_names[label_index]
        dirname = f"{root_dir}/{label_name}"
        if not os.path.exists(dirname):
            os.makedirs(dirname)
        with Image.fromarray(data) as img:
            img.save(f"{dirname}/{filename}")

meta_data = unpickle("cifar-10-batches-py/batches.meta")
label_names = meta_data['label_names']

files = [f for f in glob.glob("cifar-10-batches-py" + "**/test_*", recursive=True)]
for f in files:
    data = unpickle(f)
    output_data_to_png(data, "dataset/test")

files = [f for f in glob.glob("cifar-10-batches-py" + "**/data_*", recursive=True)]
for f in files:
    data = unpickle(f)
    output_data_to_png(data, "dataset/train")

!mkdir -p dataset/ground_truth/
for label in label_names:
    files = [f for f in glob.glob(f"dataset/train/{label}/*", recursive=True)]
    for f in files[:10]:
        !cp $f dataset/ground_truth/

!aws s3 cp --recursive dataset/ground_truth s3://bucket_name/cifar-10/dataset/ground_truth

{
  "source-ref": "s3://bucket_name/cifar-10/dataset/ground_truth/trucking_rig_s_001486.png",
  "cifar-10": 0,
  "cifar-10-metadata": {
    "confidence": 0.92,
    "job-name": "labeling-job/cifar-10",
    "class-name": "truck",
    "human-annotated": "yes",
    "creation-date": "2019-10-13T22:55:30.029703",
    "type": "groundtruth/image-classification"
  }
}

├── train
|   ├── class_1
|   │   ├── image_train_1_1.png
|   │   ├── image_train_1_2.png
|   │   ...
|   |   └── image_train_xxx.png
|   ├── class_2
|   │   ├── image_train_2_1.png
|   ...
└── test
    ├── class_1
    │   ├── image_test_1_1.png
    │   ├── image_test_1_2.png
    │   ...
    |   └── image_test_xxx.png
    ├── class_2
    │   ├── image_test_2_1.png

for label in label_names:
    for kind in ['train', 'test']:
        !aws s3 cp --recursive dataset/$kind/$label s3://bucket_name/cifar-10/dataset/$kind/$label

from __future__ import print_function

import argparse
import os
import json

import tensorflow as tf
from keras import backend as K
from keras import layers
from keras import models
from keras import optimizers
from keras.preprocessing.image import ImageDataGenerator


def save(model, model_dir):
    sess = K.get_session()
    tf.saved_model.simple_save(
        sess,
        os.path.join(model_dir, 'model/1'),
        inputs={'inputs': model.input},
        outputs={t.name: t for t in model.outputs})


def save_history(history, history_dir):
    with open(os.path.join(history_dir, 'cnn_history.json'), 'w') as f:
        json.dump(history.history, f)


def train(args):
    width = 32
    height = 32
    batch_size = args.batch_size
    epochs = args.epochs
    base_dir = args.train
    train_dir = os.path.join(base_dir, 'train')
    validation_dir = os.path.join(base_dir, 'test')

    category_num = len(os.listdir(train_dir))

    model = models.Sequential()
    model.add(layers.Conv2D(32, (3, 3), padding='same', activation='relu', input_shape=(width, height, 3)))
    model.add(layers.Conv2D(32, (3, 3), activation='relu'))
    model.add(layers.MaxPooling2D((2, 2)))
    model.add(layers.Dropout(0.25))

    model.add(layers.Conv2D(64, (3, 3), activation='relu'))
    model.add(layers.Conv2D(64, (3, 3), activation='relu'))
    model.add(layers.MaxPooling2D((2, 2)))
    model.add(layers.Dropout(0.25))

    model.add(layers.Flatten())
    model.add(layers.Dense(512, activation='relu'))
    model.add(layers.Dense(category_num, activation='softmax'))

    print(model.summary())

    model.compile(loss='categorical_crossentropy',
                  optimizer=optimizers.RMSprop(lr=1e-4),
                  metrics=['acc'])

    train_datagen = ImageDataGenerator(rescale=1./255)
    validation_datagen = ImageDataGenerator(rescale=1./255)

    train_generator = train_datagen.flow_from_directory(
        train_dir,
        target_size=(width, height),
        batch_size=batch_size,
        class_mode='categorical'
    )

    validation_generator = validation_datagen.flow_from_directory(
        validation_dir,
        target_size=(width, height),
        batch_size=batch_size,
        class_mode='categorical'
    )

    # 推論結果のインデックスと対応するクラスを出力
    print(train_generator.class_indices.items())

    history = model.fit_generator(train_generator,
                                  steps_per_epoch=100,
                                  epochs=epochs,
                                  validation_data=validation_generator,
                                  validation_steps=50)
    save_history(history, args.model_dir)
    save(model, args.model_dir)


if __name__ == '__main__':
    parser = argparse.ArgumentParser()

    # hyperparameters sent by the client are passed as command-line arguments to the script
    parser.add_argument('--epochs', type=int, default=12)
    parser.add_argument('--batch-size', type=int, default=128)

    # input data and model directories
    parser.add_argument('--model-dir', type=str, default=os.environ['SM_MODEL_DIR'])
    parser.add_argument('--train', type=str, default=os.environ['SM_CHANNEL_TRAINING'])

    args, _ = parser.parse_known_args()
    train(args)

import os
import keras
import numpy as np
import sagemaker

sagemaker_session = sagemaker.Session()
bucket_name = sagemaker_session.default_bucket()
input_data = "s3://bucket_name/cifar-10/dataset"

from sagemaker.tensorflow import TensorFlow
from sagemaker import get_execution_role

role = get_execution_role()
estimator = TensorFlow(
    entry_point = "./cifar-10-cnn.py",
    role=role,
    train_instance_count=1,
    train_instance_type="ml.p3.2xlarge",
    framework_version="1.14.0",
    py_version='py3',
    script_mode=True,
    hyperparameters={'epochs': 30, 'batch-size': 32})

estimator.fit(input_data)

output_dir = estimator.model_dir.replace("/model", "/output")
!mkdir -p tmp
!rm -rf tmp/*
!aws s3 cp --recursive $output_dir tmp/
! cd tmp && tar -xvf "model.tar.gz"

import json
f = open('tmp/cnn_history.json', 'r')
history = json.load(f)
f.close()

import matplotlib.pyplot as plt

acc = history['acc']
val_acc = history['val_acc']
loss = history['loss']
val_loss = history['val_loss']

epochs = range(len(acc))

plt.plot(epochs, acc, 'bo', label='Training acc')
plt.plot(epochs, val_acc, 'b', label='Validation acc')
plt.title('Training and validation accuracy')
plt.legend()
plt.figure()

# 損失値をプロット
plt.plot(epochs, loss, 'bo', label='Training loss')
plt.plot(epochs, val_loss, 'b', label='Validation loss')
plt.title('Training and validation loss')
plt.legend()

plt.show()

predicator = estimator.deploy(instance_type='ml.m5.xlarge', initial_instance_count=1)

import numpy as np
import matplotlib.pyplot as plt
from PIL import Image

image = Image.open('dataset/ground_truth/dawn_horse_s_000709.png')

# イメージを表示
plt.imshow(image)
plt.show()

# 推論
prediction = predicator.predict(pix.reshape(1, 32, 32, 3))['predictions']
prediction = np.array(prediction)
predicted_label = prediction.argmax(axis=1)
print('The predicted labels are: {}'.format(predicted_label))

import boto3
import json
import numpy as np
from PIL import Image

runtime= boto3.client('runtime.sagemaker')
endpoint_name = "tensorflow-training-2019-10-14-02-09-37-730"

image = Image.open('dataset/ground_truth/dawn_horse_s_000709.png')
image_array = np.asarray(image.resize((32,32)))
image_array = image_array.reshape((1,) + image_array.shape)

response = runtime.invoke_endpoint(
    EndpointName=endpoint_name,
    ContentType='application/json',
    Body=json.dumps(image_array.tolist()))

result = json.loads(response['Body'].read())
np.argmax(result['predictions'][0])