skeleton data plotting

for NTU RGB+D

정성평가를 위한 Plotting
HAR
dataset
Author

dotpyo

Published

October 10, 2023

!pwd
!ls
!cd ntu_utils/parser_repo; ls; git remote -v
/home/devin/wdir/datasets/utils
mmskeleton  ntu_utils  plotting  plotting.ipynb  working.ipynb
conf.py  __pycache__  README.md  read_skeleton.py  utils.py
origin  https://github.com/Ugenteraan/NTU-RGB-Skeleton-Python.git (fetch)
origin  https://github.com/Ugenteraan/NTU-RGB-Skeleton-Python.git (push)
# basic tools
import os
import random
from typing import List, Tuple
from dataclasses import dataclass

# basic data handling tools
import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
import time

# image processing
import cv2
import imageio
from PIL import Image
import matplotlib.image as mpimg

# Modules
from ntu_utils.parser_repo.read_skeleton import read_skeleton
# 원래 용법대로 사용하면 가져와서 쓸 수가 없어서 import 해서 쓸 수 있도록 변경
# !cd parser_repo; ls; python3 read_skeleton.py --skel="/home/devin/wdir/datasets/NTU_RGB+D/1_NTU-RGB+D/1_3DSkeletons/nturgb+d_skeletons/S001C001P001R001A001.skeleton" --save
'''
GLOBAL VARIABLE
'''


@dataclass
class args:
    # wdir/datasets/NTU_RGB+D/1_NTU-RGB+D/1_3DSkeletons/nturgb+d_skeletons/S001C001P001R001A001.skeleton
    file_name = "S001C001P001R001A002"
    skel_raw = f"../NTU_RGB+D/1_NTU-RGB+D/1_3DSkeletons/nturgb+d_skeletons/{file_name}.skeleton"
    skel_processed_sub = "../processed/st-gcn-processed-data/NTU-RGB-D/xsub/train_data.npy"
    skel_processed_view = "../processed/st-gcn-processed-data/NTU-RGB-D/xview/train_data.npy"
    subject_ID = None
    video = None
    save = None

SAMPLE = read_skeleton(args)  # args : class, global variable
SAMPLE_NUMBER = int(args.file_name.split("A")[-1])

random_seed = 1
random.seed(random_seed)

# {color_name(str) : (color code B,G,R)}
colors = {
    'blue' : (255, 0, 0,),
    'green' : (0, 255, 0),
    'red' : (0, 0, 255),
    'white' : (255, 255, 255),
    'black' : (0, 0, 0,),
    'cyan' : (0,255,255),
    'magenta' : (255,0,255),
    'yellow' :  (255,255,0)
    }

# {joint_name(str) : int}
joint_annotations = {
    "base of spine" : 1,
    "middle of spine" : 2,
    "neck" : 3,
    "head" : 4,
    "left shoulder" : 5,
    "left elbow" : 6,
    "left wrist" : 7,
    "left hand" : 8,
    "right shoulder" : 9,
    "right elbow" : 10,
    "right wrist" : 11,
    "right hand" : 12,
    "left hip" : 13,
    "left knee" : 14,
    "left ankle" : 15,
    "left foot" : 16,
    "right hip" : 17,
    "right knee" : 18,
    "right ankle" : 19,
    "right foot" : 20,
    "spine" : 21,
    "tip of left hand" : 22,
    "left thumb" : 23,
    "tip of right hand" : 24,
    "right thumb": 25
}

# {int : labels(str)}
ntu_label_annotations = {
    1   :  "drink water",
    2   :  "eat meal/snack",
    3   :  "brushing teeth",
    4   :  "brushing hair",
    5   :  "drop",
    6   :  "pickup",
    7   :  "throw",
    8   :  "sitting down",
    9   :  "standing up (from sitting position)",
    10  :  "clapping",
    11  :  "reading",
    12  :  "writing",
    13  :  "tear up paper",
    14  :  "wear jacket",
    15  :  "take off jacket",
    16  :  "wear a shoe",
    17  :  "take off a shoe",
    18  :  "wear on glasses",
    19  :  "take off glasses",
    20  :  "put on a hat/cap",
    21  :  "take off a hat/cap",
    22  :  "cheer up",
    23  :  "hand waving",
    24  :  "kicking something",
    25  :  "reach into pocket",
    26  :  "hopping (one foot jumping)",
    27  :  "jump up",
    28  :  "make a phone call/answer phone",
    29  :  "playing with phone/tablet",
    30  :  "typing on a keyboard",
    31  :  "pointing to something with finger",
    32  :  "taking a selfie",
    33  :  "check time (from watch)",
    34  :  "rub two hands together",
    35  :  "nod head/bow",
    36  :  "shake head",
    37  :  "wipe face",
    38  :  "salute",
    39  :  "put the palms together",
    40  :  "cross hands in front (say stop)",
    41  :  "sneeze/cough",
    42  :  "staggering",
    43  :  "falling",
    44  :  "touch head (headache)",
    45  :  "touch chest (stomachache/heart pain)",
    46  :  "touch back (backache)",
    47  :  "touch neck (neckache)",
    48  :  "nausea or vomiting condition",
    49  :  "use a fan (with hand or paper)/feeling warm",
    50  :  "punching/slapping other person",
    51  :  "kicking other person",
    52  :  "pushing other person",
    53  :  "pat on back of other person",
    54  :  "point finger at the other person",
    55  :  "hugging other person",
    56  :  "giving something to other person",
    57  :  "touch other person's pocket",
    58  :  "handshaking",
    59  :  "walking towards each other",
    60  :  "walking apart from each other",
    61  :  "put on headphone"
}
print(f"Sample Data has {SAMPLE.keys().__len__()} keys")        # frame이 각각 다른 것 같음 
joint_number = 1
if SAMPLE[str(joint_number)].__len__() == SAMPLE[str(joint_number+1)].__len__():
    print(f"Each key has {SAMPLE['1'][0].__len__()} values : joints")
Sample Data has 158 keys
Each key has 25 values : joints
def extract_coordinates(data:dict, target_joint:str, joint_annotations:dict=joint_annotations):
    """
    Args : 
        - data : dictionary type, skeleton extension raw NTU120 data
            - keys : total SAMPLE.keys().__len__() length, MUST be string   / length is not fixed
            - each key has 25 indexes which represents 25 joints
            - each key has 3 data which represents coordinate dimension: X, Y, (Z)
        - target_joint : configuration joint of ntu set (exact name)
        - joint_annotation : dictionary, match string 'target_joint' into its number
    """
    
    # Arguments
    X, Y, Z = 0, 1, 2
    dontknow = 0    # 이중리스트임 [[값]]
    start_frame, end_frame = 0, len(data.keys())
    joint_number = joint_annotations[target_joint] - 1      # 0부터 시작하므로

    # print
    print("="*60)
    print(f"[{target_joint}]")

    for next_frame in range(1, end_frame, 10):
        print(f"frame no. {start_frame+next_frame}\t", end=" ")
        print(f"X : {data[str(start_frame+next_frame)][dontknow][joint_number][X]:.3f}", end="\t")
        print(f"Y : {data[str(start_frame+next_frame)][dontknow][joint_number][Y]:.3f}", end="\t")
        print(f"Z : {data[str(start_frame+next_frame)][dontknow][joint_number][Z]:.3f}")

# test
for joint in ['base of spine', 'middle of spine', 'head', 'right foot', 'left foot']:
    extract_coordinates(SAMPLE, joint)
    break
============================================================
[base of spine]
frame no. 1  X : 1047.710   Y : 513.642 Z : -0.191
frame no. 11     X : 1048.290   Y : 513.821 Z : -0.183
frame no. 21     X : 1048.281   Y : 511.923 Z : -0.216
frame no. 31     X : 1049.081   Y : 509.494 Z : -0.219
frame no. 41     X : 1048.423   Y : 512.998 Z : -0.216
frame no. 51     X : 1048.136   Y : 513.229 Z : -0.224
frame no. 61     X : 1048.910   Y : 511.708 Z : -0.197
frame no. 71     X : 1049.379   Y : 512.329 Z : -0.202
frame no. 81     X : 1049.375   Y : 512.324 Z : -0.237
frame no. 91     X : 1049.288   Y : 511.088 Z : -0.226
frame no. 101    X : 1048.591   Y : 510.243 Z : -0.277
frame no. 111    X : 1048.618   Y : 510.258 Z : -0.298
frame no. 121    X : 1048.800   Y : 510.808 Z : -0.230
frame no. 131    X : 1048.711   Y : 511.964 Z : -0.236
frame no. 141    X : 1049.196   Y : 511.940 Z : -0.268
frame no. 151    X : 1050.422   Y : 511.508 Z : -0.296
def check_label(sample_number:int=SAMPLE_NUMBER, ntu_label_annotations=ntu_label_annotations) -> str:
    '''
    Args:
        - label: ntu_train_label, (0~59까지의 값:int, skeleton_file_name)

    # Note: actions labelled from A1 to A60 are contained in "NTU RGB+D" (https://rose1.ntu.edu.sg/dataset/actionRecognition/)
    '''
    label_name:str = ntu_label_annotations[SAMPLE_NUMBER]
    label_name_no_blank:str = label_name.replace(" ", "_").replace("/", "_")
    return label_name_no_blank

# test 
check_label(sample_number=SAMPLE_NUMBER)   
'eat_meal_snack'
# PARAMS
# target_joints = ['base of spine', 'middle of spine', 'spine', 'head','right elbow', 'left elbow', 'tip of left hand', 'tip of right hand','right foot', 'left foot']     # temp selection
target_joints = ['base of spine', 'middle of spine', 'neck', 'head', 
                'left shoulder', 'left elbow', 'left wrist', 'left hand', 
                'right shoulder', 'right elbow', 'right wrist', 'right hand', 
                'left hip', 'left knee', 'left ankle', 'left foot', 
                'right hip', 'right knee', 'right ankle', 'right foot', 
                'spine',
                'tip of left hand', 'left thumb', 'tip of right hand', 'right thumb']     # full joints (total 25)
start_frame, dontknow = 1, 0
X, Y, Z = 0, 1, 2

joint_numbers = [joint_annotations[target_joint] - 1 for target_joint in target_joints]
Xs = [SAMPLE[str(start_frame)][dontknow][joint_number][X] for joint_number in joint_numbers]
Ys = [SAMPLE[str(start_frame)][dontknow][joint_number][Y] for joint_number in joint_numbers]
Zs = [SAMPLE[str(start_frame)][dontknow][joint_number][Z] for joint_number in joint_numbers]
# Canvas (CV2) initial setting
# canvas : [ref](https://bkshin.tistory.com/entry/OpenCV-5-%EC%B0%BD-%EA%B4%80%EB%A6%AC-%EB%B0%8F-%EC%9D%B4%EB%B2%A4%ED%8A%B8-%EC%B2%98%EB%A6%AC)
canvas_shape = (1080, 1920, 3)    # 임의 지정 가능, video 가 1920 * 1080 이므로 그 기준으로 설정
cation_classes = 120            # ref: 3.1.2, 82 daily actions, 12 health-related actions, 26 mutual actions
subjects = 106                  # ref: 3.1.3 106 distinct subject

img = np.zeros((canvas_shape), np.uint8)    # uint : 부호 없는 정수 or 0을 포함하는 양수로 uint 또는 int 뒤에 오는 숫자는 bit의 개수를 의미한다. (uint8 : 2^8개의 정수 표현 가능, 0~255)
# '''
# 'base of spine', 'middle of spine', 'spine', 'head',
# 'right elbow', 'left elbow', 'tip of left hand', 'tip of right hand',
# 'right foot', 'left foot'
# '''

# base_of_spine     = (int(Xs[0]), int(Ys[0]))  # green
# middle_of_spine   = (int(Xs[1]), int(Ys[1]))  # blue
# spine             = (int(Xs[2]), int(Ys[2]))  # green
# head              = (int(Xs[3]), int(Ys[3]))  # red
# right_elbow       = (int(Xs[4]), int(Ys[4]))  # yellow
# left_elbow        = (int(Xs[5]), int(Ys[5]))  # yellow
# tip_of_left_hand  = (int(Xs[6]), int(Ys[6]))  # magenta
# tip_of_right_hand = (int(Xs[7]), int(Ys[7]))  # magenta
# right_foot        = (int(Xs[8]), int(Ys[8]))  # cyan
# left_foot         = (int(Xs[9]), int(Ys[9]))  # cyan
# draw line (temp selection / for 10 joints)
def draw_line_10(img, Xs, Ys):
    ## Params
    base_of_spine     = (int(Xs[0]), int(Ys[0]))  # green
    middle_of_spine   = (int(Xs[1]), int(Ys[1]))  # blue
    spine             = (int(Xs[2]), int(Ys[2]))  # green
    head              = (int(Xs[3]), int(Ys[3]))  # red
    right_elbow       = (int(Xs[4]), int(Ys[4]))  # yellow
    left_elbow        = (int(Xs[5]), int(Ys[5]))  # yellow
    tip_of_left_hand  = (int(Xs[6]), int(Ys[6]))  # magenta
    tip_of_right_hand = (int(Xs[7]), int(Ys[7]))  # magenta
    right_foot        = (int(Xs[8]), int(Ys[8]))  # cyan
    left_foot         = (int(Xs[9]), int(Ys[9]))  # cyan
    
    
    ## line
    custom_thick = 3

    result = cv2.line(img, pt1=base_of_spine,    pt2=middle_of_spine,    color=colors['green'],      thickness=custom_thick)  
    result = cv2.line(img, pt1=middle_of_spine,  pt2=spine,              color=colors['blue'],       thickness=custom_thick)  
    result = cv2.line(img, pt1=spine,            pt2=head,               color=colors['red'],        thickness=custom_thick)  
    result = cv2.line(img, pt1=spine,            pt2=right_elbow,        color=colors['cyan'],       thickness=custom_thick)  
    result = cv2.line(img, pt1=spine,            pt2=left_elbow,         color=colors['cyan'],       thickness=custom_thick)  
    result = cv2.line(img, pt1=right_elbow,      pt2=tip_of_right_hand,  color=colors['yellow'],     thickness=custom_thick)  
    result = cv2.line(img, pt1=left_elbow,       pt2=tip_of_left_hand,   color=colors['yellow'],     thickness=custom_thick)  
    result = cv2.line(img, pt1=base_of_spine,    pt2=left_foot,          color=colors['magenta'],    thickness=custom_thick)  
    result = cv2.line(img, pt1=base_of_spine,    pt2=right_foot,         color=colors['magenta'],    thickness=custom_thick)  

    return result

# draw line (full joints: 25)
def draw_line(img, Xs, Ys):
    ## Args
    connects:List[Tuple[int]] = [(1, 2), (2, 21), (21, 3), (3, 4),\
                                (21, 9), (9, 10), (10, 11), (11, 12), (12, 24), (12, 25),\
                                (21, 5), (5,6), (6,7), (7,8), (8,22), (8,23),\
                                (1, 17), (17, 18), (18, 19), (19, 20),\
                                (1, 13), (13, 14), (14, 15), (15, 16)]  # 논문 기준 (1 ~ 25)
    
    rgbcodes = [colors['green'], colors['blue'], colors['red'], colors['cyan'], colors['yellow'], colors['magenta']] * 4
    custom_thick = 3
    for i, (idx1, idx2) in enumerate(connects):
        # args
        former:tuple = (int(Xs[idx1-1]), int(Ys[idx1-1]))   # list index에 맞게 idx1 -1
        latter:tuple = (int(Xs[idx2-1]), int(Ys[idx2-1]))   # list index에 맞게 idx2 -1
        
        # draw
        result = cv2.line(img, pt1=former, pt2=latter, color=rgbcodes[i], thickness=custom_thick)
    return result

# draw_line(img, Xs, Ys)
## Local Environment ##
# to prevent kernel crash

# cv2.imshow('plotting canvas', img)      # plotting canvas 창에 이미지 표시
# cv2.waitKey(0)                          # 아무 키나 누르면
# cv2.destroyAllWindows()                 # 모든 창 닫기
# GIF으로 만들 이미지 저장
PATH_images = f"./plotting/images/{args.file_name}/"
PATH_gifs = f"./plotting/gifs/"

os.makedirs(PATH_images, exist_ok=True)
os.makedirs(PATH_gifs, exist_ok=True)

# 프레임 생성
for next_frame in range(0, SAMPLE.keys().__len__()):
    # [NOTE] Global Var
    # start frame = 1, dontknow = 0
    # target_joints = ['base of spine', 'middle of spine', 'spine', 'head','right elbow', 'left elbow', 'tip of left hand', 'tip of right hand','right foot', 'left foot']     # temp selection
    
    Xs = [SAMPLE[str(start_frame+next_frame)][dontknow][joint_number][X] for joint_number in joint_numbers] 
    Ys = [SAMPLE[str(start_frame+next_frame)][dontknow][joint_number][Y] for joint_number in joint_numbers]
    Zs = [SAMPLE[str(start_frame+next_frame)][dontknow][joint_number][Z] for joint_number in joint_numbers]
    
    # draw line
    img = np.zeros((canvas_shape), np.uint8)    # 매번 초기화
    result = draw_line(img, Xs, Ys)
    rgb_result = cv2.cvtColor(result, cv2.COLOR_BGR2RGB)

    try:
        cv2.imwrite(f'{PATH_images}/{start_frame+next_frame}.jpg', result)
    except: break
path = [f"{PATH_images}/{i}" for i in os.listdir(PATH_images)]
paths = [ Image.open(i) for i in path]

imageio.mimsave(f'{PATH_gifs}/{args.file_name}_{check_label(sample_number=SAMPLE_NUMBER)}_{len(Xs)}.gif', paths, fps=5)

NTU RGB+D

임의의 joint 10개 추출 25개 joint 전부 Plotting
S001C001P001R001A001 drink water
S001C001P001R001A001_drink_water S001C001P001R001A001_drink_water_25
S001C001P001R001A002 eat_meal/snack
S001C001P001R001A002_eat_meal_snack S001C001P001R001A002_eat_meal_snack_25
S001C001P001R001A003 brushing_teeth
S001C001P001R001A003_brushing_teeth S001C001P001R001A003_brushing_teeth_25