I’m developing an app with React-Native for Android. The app was supposed to use the ML Kit PoseDetection API of Google to count reps of an exercise. Right now I just want the app to open the camera, use the detection on me and draw lines on the camera image so that I know the pose detection works. I used the following article of medium.com as a model: https://medium.com/dogtronic/real-time-pose-detection-in-react-native-using-mlkit-e1819847c340
The whole code is written for iOS, so I had to make a few adjustments for the android system. I admit: I used ChatGPT for this because I’m not familiar with Java, Kotlin etc.
The gradle build is always successful.
First my package.json:
{
"name": "PoseDetection",
"displayName": "PoseDetection",
"version": "0.0.1",
"private": true,
"peerDependencies": {
"react-native-worklets-core": "1.3.3"
},
"scripts": {
"android": "react-native run-android",
"ios": "react-native run-ios",
"lint": "eslint .",
"start": "react-native start",
"test": "jest"
},
"dependencies": {
"metro-react-native-babel-preset": "^0.77.0",
"react": "18.2.0",
"react-native": "0.73.6",
"react-native-gesture-handler": "^2.14.0",
"react-native-reanimated": "^3.6.2",
"react-native-svg": "^15.3.0",
"react-native-vision-camera": "^4.0.5",
"react-native-worklets-core": "^1.3.3"
},
"devDependencies": {
"@babel/core": "^7.20.0",
"@babel/plugin-proposal-class-properties": "^7.18.6",
"@babel/plugin-transform-private-methods": "^7.24.7",
"@babel/preset-env": "^7.20.0",
"@babel/runtime": "^7.20.0",
"@react-native/babel-preset": "0.74.84",
"@react-native/eslint-config": "0.74.84",
"@react-native/metro-config": "0.74.84",
"@react-native/typescript-config": "0.74.84",
"@types/react": "^18.2.6",
"@types/react-test-renderer": "^18.0.0",
"babel-jest": "^29.6.3",
"eslint": "^8.19.0",
"jest": "^29.6.3",
"prettier": "2.8.8",
"react-test-renderer": "18.2.0",
"typescript": "5.0.4"
},
"engines": {
"node": ">=18"
}
}
and the main part of build.gradle in androidapp:
android {
ndkVersion rootProject.ext.ndkVersion
buildToolsVersion rootProject.ext.buildToolsVersion
compileSdk rootProject.ext.compileSdkVersion
namespace "com.posedetection"
defaultConfig {
applicationId "com.posedetection"
minSdkVersion rootProject.ext.minSdkVersion
targetSdkVersion rootProject.ext.targetSdkVersion
versionCode 1
versionName "1.0"
}
signingConfigs {
debug {
storeFile file('debug.keystore')
storePassword 'android'
keyAlias 'androiddebugkey'
keyPassword 'android'
}
}
buildTypes {
debug {
signingConfig signingConfigs.debug
}
release {
// Caution! In production, you need to generate your own keystore file.
// see https://reactnative.dev/docs/signed-apk-android.
signingConfig signingConfigs.debug
minifyEnabled enableProguardInReleaseBuilds
proguardFiles getDefaultProguardFile("proguard-android.txt"), "proguard-rules.pro"
}
}
}
dependencies {
implementation("com.facebook.react:react-android")
implementation("com.facebook.react:react-native:+")
implementation("com.facebook.soloader:soloader:0.10.3")
implementation 'com.google.mlkit:pose-detection:18.0.0-beta4'
implementation 'com.google.mlkit:pose-detection-accurate:18.0.0-beta4'
implementation 'com.google.mlkit:vision-common:17.0.0'
implementation 'com.google.mlkit:vision-interfaces:16.0.0'
implementation 'com.google.mlkit:camera:16.0.0-beta3'
implementation 'androidx.annotation:annotation:1.3.0'
implementation 'com.google.android.gms:play-services-tasks:18.0.0'
implementation 'com.android.support:multidex:1.0.3'
implementation project(':react-native-gesture-handler')
implementation project(':react-native-vision-camera')
implementation project(':react-native-svg')
implementation project(':react-native-worklets-core')
if (hermesEnabled.toBoolean()) {
implementation("com.facebook.react:hermes-android")
} else {
implementation jscFlavor
}
}
Like I said, I used ChatGPT most of the time. It told me to create 2 files:
PoseDetectionModule.java:
public class PoseDetectionModule extends ReactContextBaseJavaModule {
private final PoseDetector poseDetector;
public PoseDetectionModule(ReactApplicationContext reactContext) {
super(reactContext);
PoseDetectorOptions options = new PoseDetectorOptions.Builder()
.setDetectorMode(PoseDetectorOptions.STREAM_MODE)
.build();
poseDetector = PoseDetection.getClient(options);
}
@Override
public String getName() {
return "PoseDetection";
}
@ReactMethod
public void detectPose(Bitmap bitmap, Promise promise) {
InputImage image = InputImage.fromBitmap(bitmap, 0);
poseDetector.process(image)
.addOnSuccessListener(
pose -> {
WritableMap poseData = Arguments.createMap();
List<PoseLandmark> landmarks = pose.getAllPoseLandmarks();
for (PoseLandmark landmark : landmarks) {
String landmarkName = getLandmarkTypeString(landmark.getLandmarkType());
WritableMap position = Arguments.createMap();
PointF3D position3D = landmark.getPosition3D();
position.putDouble("x", position3D.getX());
position.putDouble("y", position3D.getY());
position.putDouble("z", position3D.getZ());
poseData.putMap(landmarkName, position);
}
promise.resolve(poseData);
})
.addOnFailureListener(
e -> {
promise.reject("Pose Detection Error", e);
});
}
private String getLandmarkTypeString(int landmarkType) {
switch (landmarkType) {
case PoseLandmark.NOSE:
return "NOSE";
case PoseLandmark.LEFT_EYE_INNER:
return "LEFT_EYE_INNER";
case PoseLandmark.LEFT_EYE:
return "LEFT_EYE";
case PoseLandmark.LEFT_EYE_OUTER:
return "LEFT_EYE_OUTER";
case PoseLandmark.RIGHT_EYE_INNER:
return "RIGHT_EYE_INNER";
case PoseLandmark.RIGHT_EYE:
return "RIGHT_EYE";
case PoseLandmark.RIGHT_EYE_OUTER:
return "RIGHT_EYE_OUTER";
case PoseLandmark.LEFT_EAR:
return "LEFT_EAR";
case PoseLandmark.RIGHT_EAR:
return "RIGHT_EAR";
case PoseLandmark.LEFT_MOUTH:
return "LEFT_MOUTH";
case PoseLandmark.RIGHT_MOUTH:
return "RIGHT_MOUTH";
case PoseLandmark.LEFT_SHOULDER:
return "LEFT_SHOULDER";
case PoseLandmark.RIGHT_SHOULDER:
return "RIGHT_SHOULDER";
case PoseLandmark.LEFT_ELBOW:
return "LEFT_ELBOW";
case PoseLandmark.RIGHT_ELBOW:
return "RIGHT_ELBOW";
case PoseLandmark.LEFT_WRIST:
return "LEFT_WRIST";
case PoseLandmark.RIGHT_WRIST:
return "RIGHT_WRIST";
case PoseLandmark.LEFT_PINKY:
return "LEFT_PINKY";
case PoseLandmark.RIGHT_PINKY:
return "RIGHT_PINKY";
case PoseLandmark.LEFT_INDEX:
return "LEFT_INDEX";
case PoseLandmark.RIGHT_INDEX:
return "RIGHT_INDEX";
case PoseLandmark.LEFT_THUMB:
return "LEFT_THUMB";
case PoseLandmark.RIGHT_THUMB:
return "RIGHT_THUMB";
case PoseLandmark.LEFT_HIP:
return "LEFT_HIP";
case PoseLandmark.RIGHT_HIP:
return "RIGHT_HIP";
case PoseLandmark.LEFT_KNEE:
return "LEFT_KNEE";
case PoseLandmark.RIGHT_KNEE:
return "RIGHT_KNEE";
case PoseLandmark.LEFT_ANKLE:
return "LEFT_ANKLE";
case PoseLandmark.RIGHT_ANKLE:
return "RIGHT_ANKLE";
case PoseLandmark.LEFT_HEEL:
return "LEFT_HEEL";
case PoseLandmark.RIGHT_HEEL:
return "RIGHT_HEEL";
case PoseLandmark.LEFT_FOOT_INDEX:
return "LEFT_FOOT_INDEX";
case PoseLandmark.RIGHT_FOOT_INDEX:
return "RIGHT_FOOT_INDEX";
default:
return "UNKNOWN";
}
}
}
PoseDetectionPackage.java:
public class PoseDetectionPackage implements ReactPackage {
@Override
public List<ViewManager> createViewManagers(ReactApplicationContext reactContext) {
return Collections.emptyList();
}
@Override
public List<NativeModule> createNativeModules(
ReactApplicationContext reactContext) {
List<NativeModule> modules = new ArrayList<>();
modules.add(new PoseDetectionModule(reactContext));
return modules;
}
}
I returned the package I created in MainApplication.kt:
package com.posedetection
import android.app.Application
import com.facebook.react.ReactApplication
import com.facebook.react.ReactNativeHost
import com.facebook.react.ReactPackage
import com.facebook.react.shell.MainReactPackage
import com.facebook.soloader.SoLoader
import com.swmansion.gesturehandler.RNGestureHandlerPackage
import com.swmansion.reanimated.ReanimatedPackage
import com.posedetection.PoseDetectionPackage
import java.util.Arrays
import com.mrousavy.camera.react.CameraPackage;
import com.horcrux.svg.SvgPackage
import com.worklets.WorkletsPackage
class MainApplication : Application(), ReactApplication {
private val mReactNativeHost: ReactNativeHost = object : ReactNativeHost(this) {
override fun getUseDeveloperSupport(): Boolean {
return BuildConfig.DEBUG
}
override fun getPackages(): List<ReactPackage> {
return Arrays.asList<ReactPackage>(
MainReactPackage(),
PoseDetectionPackage(), // Füge diese Zeile hinzu
RNGestureHandlerPackage(),
ReanimatedPackage(),
CameraPackage(),
SvgPackage(),
WorkletsPackage()
)
}
override fun getJSMainModuleName(): String {
return "index"
}
}
override val reactNativeHost: ReactNativeHost
get() = mReactNativeHost
o }verride fun onCreate() {
super.onCreate()
SoLoader.init(this, /* native exopackage */ false)
// for using flipper just in case
}
}
And then there is my App.tsx in my project folder:
import React, { useEffect } from 'react';
import { Dimensions, StyleSheet, Text, useWindowDimensions } from 'react-native';
import { Camera, useFrameProcessor, useCameraDevice } from 'react-native-vision-camera';
import Animated, { useAnimatedProps } from 'react-native-reanimated';
import Svg, { Line } from 'react-native-svg';
import { useSharedValue } from 'react-native-worklets-core';
import { NativeModules } from 'react-native';
const { PoseDetection } = NativeModules;
const AnimatedLine = Animated.createAnimatedComponent(Line);
export function __poseDetection(frame: any): any {
'worklet';
return new Promise((resolve) => {
PoseDetection.detectPose(frame, (result: any) => {
resolve(result);
});
});
}
const usePosition = (pose: any, valueName1: string, valueName2: string) => {
return useAnimatedProps(
() => ({
x1: pose.value[valueName1]?.x ?? 0,
y1: pose.value[valueName1]?.y ?? 0,
x2: pose.value[valueName2]?.x ?? 0,
y2: pose.value[valueName2]?.y ?? 0,
}),
[pose],
);
};
const defaultPose = {
leftShoulder: { x: 0, y: 0 },
rightShoulder: { x: 0, y: 0 },
leftElbow: { x: 0, y: 0 },
rightElbow: { x: 0, y: 0 },
leftWrist: { x: 0, y: 0 },
rightWrist: { x: 0, y: 0 },
leftHip: { x: 0, y: 0 },
rightHip: { x: 0, y: 0 },
leftKnee: { x: 0, y: 0 },
rightKnee: { x: 0, y: 0 },
leftAnkle: { x: 0, y: 0 },
rightAnkle: { x: 0, y: 0 },
};
const App = () => {
const pose = useSharedValue(defaultPose);
const leftWristToElbowPosition = usePosition(pose, 'leftWrist', 'leftElbow');
const leftElbowToShoulderPosition = usePosition(pose, 'leftElbow', 'leftShoulder');
const leftShoulderToHipPosition = usePosition(pose, 'leftShoulder', 'leftHip');
const leftHipToKneePosition = usePosition(pose, 'leftHip', 'leftKnee');
const leftKneeToAnklePosition = usePosition(pose, 'leftKnee', 'leftAnkle');
const rightWristToElbowPosition = usePosition(pose, 'rightWrist', 'rightElbow');
const rightElbowToShoulderPosition = usePosition(pose, 'rightElbow', 'rightShoulder');
const rightShoulderToHipPosition = usePosition(pose, 'rightShoulder', 'rightHip');
const rightHipToKneePosition = usePosition(pose, 'rightHip', 'rightKnee');
const rightKneeToAnklePosition = usePosition(pose, 'rightKnee', 'rightAnkle');
const shoulderToShoulderPosition = usePosition(pose, 'leftShoulder', 'rightShoulder');
const hipToHipPosition = usePosition(pose, 'leftHip', 'rightHip');
const dimensions = useWindowDimensions();
const frameProcessor = useFrameProcessor((frame) => {
'worklet';
const poseObject = __poseDetection(frame);
const xFactor = dimensions.width / frame.width;
const yFactor = dimensions.height / frame.height;
const poseCopy = { ...defaultPose };
Object.keys(poseObject).forEach((v) => {
poseCopy[v] = {
x: poseObject[v].x * xFactor,
y: poseObject[v].y * yFactor,
};
});
pose.value = poseCopy;
}, []);
const device = useCameraDevice('front');
useEffect(() => {
const checkPermissions = async () => {
await Camera.requestCameraPermission();
};
checkPermissions();
}, []);
if (device == null) {
return <Text>Loading...</Text>;
}
return (
<>
<Camera
frameProcessor={frameProcessor}
style={StyleSheet.absoluteFill}
device={device}
isActive={true}
/>
<Svg
height={Dimensions.get('window').height}
width={Dimensions.get('window').width}
style={styles.linesContainer}>
<AnimatedLine animatedProps={leftWristToElbowPosition} stroke="red" strokeWidth="2" />
<AnimatedLine animatedProps={leftElbowToShoulderPosition} stroke="red" strokeWidth="2" />
<AnimatedLine animatedProps={leftShoulderToHipPosition} stroke="red" strokeWidth="2" />
<AnimatedLine animatedProps={leftHipToKneePosition} stroke="red" strokeWidth="2" />
<AnimatedLine animatedProps={leftKneeToAnklePosition} stroke="red" strokeWidth="2" />
<AnimatedLine animatedProps={rightWristToElbowPosition} stroke="red" strokeWidth="2" />
<AnimatedLine animatedProps={rightElbowToShoulderPosition} stroke="red" strokeWidth="2" />
<AnimatedLine animatedProps={rightShoulderToHipPosition} stroke="red" strokeWidth="2" />
<AnimatedLine animatedProps={rightHipToKneePosition} stroke="red" strokeWidth="2" />
<AnimatedLine animatedProps={rightKneeToAnklePosition} stroke="red" strokeWidth="2" />
<AnimatedLine animatedProps={shoulderToShoulderPosition} stroke="red" strokeWidth="2" />
<AnimatedLine animatedProps={hipToHipPosition} stroke="red" strokeWidth="2" />
</Svg>
</>
);
};
const styles = StyleSheet.create({
linesContainer: {
position: 'absolute',
top: 0,
left: 0,
height: Dimensions.get('window').height,
width: Dimensions.get('window').width,
},
});
export default App;
I know it’s a lot of code but I think they are necessary for my problem.
I get the following error:
Why does it say it doesn’t exist? I can’t tell if my pose detection just simply doesn’t work, the function is coded wrong or I just imported something wrong.
I tried to define ‘__poseDetection’ in a different way to see if there is a logic issue, but the error still appears no matter what I do
I checked the import of the PoseDetectionModule in App.tsx according to the directions of React Native and it seems to be fine
I tried it by adding debugging commands, but I get nothing useful or even nothing
I will probably create a post on GitHub as well because this issue was supposed to be a project for my university so it’s quite important