dnutiu/ImageTagger
dnutiu/ImageTagger
1
1
Fork 0
Code Issues Pull requests Projects Releases 2 Packages Wiki Activity Actions

extract common ai-related functionality into img-ai

Browse source
This commit is contained in:
Denis-Cosmin NUTIU 2024-03-30 12:11:39 +02:00
parent cdc4214789
commit 3b717d3861
12 changed files with 2137 additions and 17 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

1
.idea/gradle.xml
View file

@ -8,6 +8,7 @@
<option name="modules">
<set>
<option value="$PROJECT_DIR$" />
<option value="$PROJECT_DIR$/img-ai" />
<option value="$PROJECT_DIR$/img-ui" />
</set>
</option>

118
.idea/workspace.xml
View file

@ -4,16 +4,17 @@
<option name="autoReloadType" value="SELECTIVE" />
</component>
<component name="ChangeListManager">
<list default="true" id="dac8f85c-5bd6-4e05-9520-9e6a91c7f78e" name="Changes" comment="update readme.md">
<change afterPath="$PROJECT_DIR$/settings.gradle" afterDir="false" />
<change beforePath="$PROJECT_DIR$/.idea/gradle.xml" beforeDir="false" afterPath="$PROJECT_DIR$/.idea/gradle.xml" afterDir="false" />
<change beforePath="$PROJECT_DIR$/.idea/misc.xml" beforeDir="false" afterPath="$PROJECT_DIR$/.idea/misc.xml" afterDir="false" />
<list default="true" id="dac8f85c-5bd6-4e05-9520-9e6a91c7f78e" name="Changes" comment="update gradle config">
<change afterPath="$PROJECT_DIR$/img-ai/.gitignore" afterDir="false" />
<change afterPath="$PROJECT_DIR$/img-ai/src/main/kotlin/ImageTagsPrediction.kt" afterDir="false" />
<change afterPath="$PROJECT_DIR$/img-ai/src/main/resources/dev/nuculabs/imagetagger/ai/prediction_categories.txt" afterDir="false" />
<change afterPath="$PROJECT_DIR$/img-ai/src/test/kotlin/ImageTagsPredictionTests.kt" afterDir="false" />
<change afterPath="$PROJECT_DIR$/img-ai/src/test/resources/dev/nuculabs/imagetagger/ai/timisoara-bega.jpg" afterDir="false" />
<change afterPath="$PROJECT_DIR$/img-ai/src/test/resources/dev/nuculabs/imagetagger/ai/timisoara-threes.jpg" afterDir="false" />
<change afterPath="$PROJECT_DIR$/img-ai/src/test/resources/dev/nuculabs/imagetagger/ai/timisoara-water-tower.jpg" afterDir="false" />
<change beforePath="$PROJECT_DIR$/.idea/workspace.xml" beforeDir="false" afterPath="$PROJECT_DIR$/.idea/workspace.xml" afterDir="false" />
<change beforePath="$PROJECT_DIR$/img-ui/gradle/wrapper/gradle-wrapper.jar" beforeDir="false" />
<change beforePath="$PROJECT_DIR$/img-ui/gradle/wrapper/gradle-wrapper.properties" beforeDir="false" />
<change beforePath="$PROJECT_DIR$/img-ui/gradlew" beforeDir="false" afterPath="$PROJECT_DIR$/img-ui/gradlew" afterDir="false" />
<change beforePath="$PROJECT_DIR$/img-ui/gradlew.bat" beforeDir="false" afterPath="$PROJECT_DIR$/img-ui/gradlew.bat" afterDir="false" />
<change beforePath="$PROJECT_DIR$/img-ui/settings.gradle" beforeDir="false" />
<change beforePath="$PROJECT_DIR$/img-ui/build.gradle" beforeDir="false" afterPath="$PROJECT_DIR$/img-ui/build.gradle" afterDir="false" />
<change beforePath="$PROJECT_DIR$/settings.gradle" beforeDir="false" afterPath="$PROJECT_DIR$/settings.gradle" afterDir="false" />
</list>
<option name="SHOW_DIALOG" value="false" />
<option name="HIGHLIGHT_CONFLICTS" value="true" />
@ -40,6 +41,9 @@
<task path="$PROJECT_DIR$/ImageTagger">
<activation />
</task>
<task path="$PROJECT_DIR$/img-ai">
<activation />
</task>
<projects_view>
<tree_state>
<expand>
@ -47,6 +51,24 @@
<item name="" type="6a2764b6:ExternalProjectsStructure$RootNode" />
<item name="ImageTagger-Solution" type="f1a62948:ProjectNode" />
</path>
<path>
<item name="" type="6a2764b6:ExternalProjectsStructure$RootNode" />
<item name="ImageTagger-Solution" type="f1a62948:ProjectNode" />
<item name="img-ai" type="2d1252cf:ModuleNode" />
</path>
<path>
<item name="" type="6a2764b6:ExternalProjectsStructure$RootNode" />
<item name="ImageTagger-Solution" type="f1a62948:ProjectNode" />
<item name="img-ai" type="2d1252cf:ModuleNode" />
<item name="Tasks" type="e4a08cd1:TasksNode" />
</path>
<path>
<item name="" type="6a2764b6:ExternalProjectsStructure$RootNode" />
<item name="ImageTagger-Solution" type="f1a62948:ProjectNode" />
<item name="img-ai" type="2d1252cf:ModuleNode" />
<item name="Tasks" type="e4a08cd1:TasksNode" />
<item name="verification" type="c8890929:TasksNode$1" />
</path>
</expand>
<select />
</tree_state>
@ -72,6 +94,7 @@
<component name="PropertiesComponent"><![CDATA[{
"keyToString": {
"Gradle.Build img-ui.executor": "Run",
"Gradle.ImageTagger:img-ai [test].executor": "Run",
"Gradle.img-ui [build].executor": "Run",
"Gradle.img-ui [run].executor": "Run",
"Kotlin.MainPageKt.executor": "Debug",
@ -79,14 +102,15 @@
"RunOnceActivity.ShowReadmeOnStart": "true",
"dart.analysis.tool.window.visible": "false",
"git-widget-placeholder": "multi-project",
"jdk.selected.JAVA_MODULE": "17",
"kotlin-language-version-configured": "true",
"last_opened_file_path": "/Users/dnutiu/Projects/ImageTagger/img-ui",
"last_opened_file_path": "/Users/dnutiu/Projects/ImageTagger/img-ai/src/test/resources/dev/nuculabs/imagetagger/ai",
"node.js.detected.package.eslint": "true",
"node.js.detected.package.tslint": "true",
"node.js.selected.package.eslint": "(autodetect)",
"node.js.selected.package.tslint": "(autodetect)",
"nodejs_package_manager_path": "npm",
"project.structure.last.edited": "SDKs",
"project.structure.last.edited": "Modules",
"project.structure.proportion": "0.15",
"project.structure.side.proportion": "0.29070836",
"settings.editor.selected.configurable": "reference.settingsdialog.project.gradle",
@ -95,14 +119,66 @@
}]]></component>
<component name="RecentsManager">
<key name="CopyFile.RECENT_KEYS">
<recent name="$PROJECT_DIR$/img-ai/src/test/resources/dev/nuculabs/imagetagger/ai" />
<recent name="$PROJECT_DIR$/img-ai/src/main/resources/dev.nuculabs.imagetagger.ai" />
<recent name="$PROJECT_DIR$/img-ui" />
<recent name="$PROJECT_DIR$" />
</key>
<key name="MoveFile.RECENT_KEYS">
<recent name="$PROJECT_DIR$/img-ai/src/main/resources/dev/nuculabs/imagetagger/ai" />
<recent name="$PROJECT_DIR$" />
</key>
<key name="CopyKotlinDeclarationDialog.RECENTS_KEY">
<recent name="" />
</key>
</component>
<component name="RunManager" selected="Gradle.img-ui [run]">
<component name="RunManager" selected="Gradle.ImageTagsPredictionTests">
<configuration name="ImageTagger:img-ai [test]" type="GradleRunConfiguration" factoryName="Gradle" temporary="true">
<ExternalSystemSettings>
<option name="executionName" />
<option name="externalProjectPath" value="$PROJECT_DIR$/img-ai" />
<option name="externalSystemIdString" value="GRADLE" />
<option name="scriptParameters" />
<option name="taskDescriptions">
<list />
</option>
<option name="taskNames">
<list>
<option value="test" />
</list>
</option>
<option name="vmOptions" />
</ExternalSystemSettings>
<ExternalSystemDebugServerProcess>true</ExternalSystemDebugServerProcess>
<ExternalSystemReattachDebugProcess>true</ExternalSystemReattachDebugProcess>
<DebugAllEnabled>false</DebugAllEnabled>
<RunAsTest>false</RunAsTest>
<method v="2" />
</configuration>
<configuration name="ImageTagsPredictionTests" type="GradleRunConfiguration" factoryName="Gradle" temporary="true">
<ExternalSystemSettings>
<option name="executionName" />
<option name="externalProjectPath" value="$PROJECT_DIR$" />
<option name="externalSystemIdString" value="GRADLE" />
<option name="scriptParameters" value="" />
<option name="taskDescriptions">
<list />
</option>
<option name="taskNames">
<list>
<option value=":img-ai:test" />
<option value="--tests" />
<option value="&quot;dev.nuculabs.imagetagger.ai.ImageTagsPredictionTests&quot;" />
</list>
</option>
<option name="vmOptions" />
</ExternalSystemSettings>
<ExternalSystemDebugServerProcess>false</ExternalSystemDebugServerProcess>
<ExternalSystemReattachDebugProcess>true</ExternalSystemReattachDebugProcess>
<DebugAllEnabled>false</DebugAllEnabled>
<RunAsTest>true</RunAsTest>
<method v="2" />
</configuration>
<configuration name="img-ui [build]" type="GradleRunConfiguration" factoryName="Gradle" temporary="true">
<ExternalSystemSettings>
<option name="executionName" />
@ -149,6 +225,8 @@
</configuration>
<recent_temporary>
<list>
<item itemvalue="Gradle.ImageTagsPredictionTests" />
<item itemvalue="Gradle.ImageTagger:img-ai [test]" />
<item itemvalue="Gradle.img-ui [run]" />
<item itemvalue="Gradle.img-ui [build]" />
</list>
@ -157,6 +235,7 @@
<component name="SharedIndexes">
<attachedChunks>
<set>
<option value="jdk-17.0.10-corretto-17.0.10-4caba194b151-53826d6a" />
<option value="jdk-17.0.9-corretto-17.0.9-4caba194b151-92e50af3" />
</set>
</attachedChunks>
@ -170,7 +249,7 @@
<option name="presentableId" value="Default" />
<updated>1711789328334</updated>
<workItem from="1711789330084" duration="2109000" />
<workItem from="1711791444105" duration="348000" />
<workItem from="1711791444105" duration="1766000" />
</task>
<task id="LOCAL-00001" summary="split ImageTagger into subproject img-ui">
<option name="closed" value="true" />
@ -188,7 +267,15 @@
<option name="project" value="LOCAL" />
<updated>1711790617285</updated>
</task>
<option name="localTasksCounter" value="3" />
<task id="LOCAL-00003" summary="update gradle config">
<option name="closed" value="true" />
<created>1711791812195</created>
<option name="number" value="00003" />
<option name="presentableId" value="LOCAL-00003" />
<option name="project" value="LOCAL" />
<updated>1711791812195</updated>
</task>
<option name="localTasksCounter" value="4" />
<servers />
</component>
<component name="TypeScriptGeneratedFilesManager">
@ -197,7 +284,8 @@
<component name="VcsManagerConfiguration">
<MESSAGE value="split ImageTagger into subproject img-ui" />
<MESSAGE value="update readme.md" />
<option name="LAST_COMMIT_MESSAGE" value="update readme.md" />
<MESSAGE value="update gradle config" />
<option name="LAST_COMMIT_MESSAGE" value="update gradle config" />
</component>
<component name="XDebuggerManager">
<breakpoint-manager>

2
img-ai/.gitignore vendored Normal file
View file

@ -0,0 +1,2 @@
# Deep learning models
src/main/resources/dev/nuculabs/imagetagger/ai/*.onnx

22
img-ai/build.gradle.kts Normal file
View file

@ -0,0 +1,22 @@
plugins {
kotlin("jvm") version "1.8.22"
}
group = "dev.nuculabs.imagetagger.ai"
version = "1.1"
repositories {
mavenCentral()
}
dependencies {
implementation("com.microsoft.onnxruntime:onnxruntime:1.17.1")
testImplementation("org.jetbrains.kotlin:kotlin-test")
}
tasks.test {
useJUnitPlatform()
}
kotlin {
jvmToolchain(17)
}

173
img-ai/src/main/kotlin/ImageTagsPrediction.kt Normal file
View file

@ -0,0 +1,173 @@
package dev.nuculabs.imagetagger.ai
import ai.onnxruntime.OnnxTensor
import ai.onnxruntime.OrtEnvironment
import ai.onnxruntime.OrtSession
import java.awt.image.BufferedImage
import java.io.IOException
import java.io.InputStream
import java.util.logging.Logger
import javax.imageio.ImageIO
/**
* ImageTagsPrediction is a specialized class that predicts an Image's tags
*/
class ImageTagsPrediction {
private val logger: Logger = Logger.getLogger("InfoLogging")
private var ortEnv: OrtEnvironment = OrtEnvironment.getEnvironment()
private var ortSession: OrtSession
private var modelClasses: MutableList<String> = mutableListOf()
init {
try {
logger.info("Loading ML model. Please wait.")
ImageTagsPrediction::class.java.getResourceAsStream("/dev/nuculabs/imagetagger/ai/prediction.onnx").let { modelFile ->
ortSession = ortEnv.createSession(
modelFile!!.readBytes(),
OrtSession.SessionOptions()
)
}
ImageTagsPrediction::class.java.getResourceAsStream("/dev/nuculabs/imagetagger/ai/prediction_categories.txt")
.let { classesFile ->
modelClasses.addAll(0, classesFile!!.bufferedReader().readLines())
}
logger.info("Loaded ${modelClasses.size} model classes.")
} catch (e: NullPointerException) {
logger.severe(
"Failed to load model file or categories file. If you're building the project from " +
"source, please follow the instructions from the README.md: " +
"https://github.com/dnutiu/ImageTagger." +
"Exception ${e.message}"
)
throw e
}
}
/**
* Processes an image into an ONNX Tensor.
*/
private fun processImage(bufferedImage: BufferedImage): Array<Array<Array<FloatArray>>> {
try {
val tensorData = Array(1) {
Array(3) {
Array(224) {
FloatArray(224)
}
}
}
val mean = floatArrayOf(0.485f, 0.456f, 0.406f)
val standardDeviation = floatArrayOf(0.229f, 0.224f, 0.225f)
// crop image to 224x224
var width: Int = bufferedImage.width
var height: Int = bufferedImage.height
var startX = 0
var startY = 0
if (width > height) {
startX = (width - height) / 2
width = height
} else {
startY = (height - width) / 2
height = width
}
val image = bufferedImage.getSubimage(startX, startY, width, height)
val resizedImage = image.getScaledInstance(224, 224, 4)
val scaledImage = BufferedImage(224, 224, BufferedImage.TYPE_4BYTE_ABGR)
scaledImage.graphics.drawImage(resizedImage, 0, 0, null)
// Process image
for (y in 0 until scaledImage.height) {
for (x in 0 until scaledImage.width) {
val pixel: Int = scaledImage.getRGB(x, y)
// Get RGB values
tensorData[0][0][y][x] =
((pixel shr 16 and 0xFF) / 255f - mean[0]) / standardDeviation[0]
tensorData[0][1][y][x] =
((pixel shr 16 and 0xFF) / 255f - mean[1]) / standardDeviation[1]
tensorData[0][2][y][x] =
((pixel shr 16 and 0xFF) / 255f - mean[2]) / standardDeviation[2]
}
}
return tensorData
} catch (e: IOException) {
throw RuntimeException(e)
}
}
/**
* Uses the ML model to predict tags for a given bitmap.
*/
@Suppress("UNCHECKED_CAST")
private fun predictTagsInternal(bufferedImage: BufferedImage): List<String> {
// 1. Get input and output names
val inputName: String = ortSession.inputNames.iterator().next()
val outputName: String = ortSession.outputNames.iterator().next()
// 2. Create input tensor
val inputTensor = OnnxTensor.createTensor(ortEnv, processImage(bufferedImage))
// 3. Run the model.
val inputs = mapOf(inputName to inputTensor)
val results = ortSession.run(inputs)
// 4. Get output tensor
val outputTensor = results.get(outputName)
if (outputTensor.isPresent) {
// 5. Get prediction results
val floatBuffer = outputTensor.get().value as Array<FloatArray>
val predictions = ArrayList<String>()
// filter buffer by threshold
for (i in floatBuffer[0].indices) {
if (floatBuffer[0][i] > -0.5) {
predictions.add(modelClasses[i])
}
}
return predictions
} else {
return ArrayList()
}
}
/**
* Predicts tags for a Bitmap.
*/
fun predictTags(image: BufferedImage): List<String> {
return predictTagsInternal(image)
}
/**
* Predicts tags for a given image input stream.
*/
fun predictTags(input: InputStream?): List<String> {
if (input == null) {
return ArrayList()
}
return predictTagsInternal(ImageIO.read(input))
}
/**
* Close the session and environment.
*/
fun close() {
ortSession.close()
ortEnv.close()
modelClasses.clear()
}
// Singleton Pattern
companion object {
@Volatile
private var instance: ImageTagsPrediction? = null
fun getInstance() =
instance ?: synchronized(this) {
instance ?: ImageTagsPrediction().also { instance = it }
}
}
}

1708
img-ai/src/main/resources/dev/nuculabs/imagetagger/ai/prediction_categories.txt Normal file
View file

File diff suppressed because it is too large Load diff

121
img-ai/src/test/kotlin/ImageTagsPredictionTests.kt Normal file
View file

@ -0,0 +1,121 @@
package dev.nuculabs.imagetagger.ai
import org.junit.jupiter.api.AfterAll
import org.junit.jupiter.api.Assertions.assertEquals
import org.junit.jupiter.api.BeforeAll
import org.junit.jupiter.api.Test
import java.io.File
import javax.imageio.ImageIO
class ImageTagsPredictionTests {
@Test
fun testPredictTagsForBufferedImage_TimisoaraBega() {
val timisoaraBega = ImageTagsPredictionTests::class.java.getResource("timisoara-bega.jpg")
val tags = imageTagsPrediction.predictTags(ImageIO.read(timisoaraBega))
assertEquals(
listOf(
"lake",
"nature",
"no people",
"outdoors",
"reflection",
"river",
"sky",
"tranquil",
"tree",
"water"
), tags
)
}
@Test
fun testPredictTagsForBufferedImage_TimisoaraThrees() {
val timisoaraBega = ImageTagsPredictionTests::class.java.getResource("timisoara-threes.jpg")
val tags = imageTagsPrediction.predictTags(ImageIO.read(timisoaraBega))
assertEquals(
listOf("day", "forest", "growth", "nature", "no people", "outdoors", "plant", "tree"), tags
)
}
@Test
fun testPredictTagsForBufferedImage_TimisoaraWaterTower() {
val timisoaraBega = ImageTagsPredictionTests::class.java.getResource("timisoara-water-tower.jpg")
val tags = imageTagsPrediction.predictTags(ImageIO.read(timisoaraBega))
assertEquals(
listOf(
"architecture",
"building exterior",
"built structure",
"day",
"history",
"no people",
"outdoors",
"travel destinations"
), tags
)
}
@Test
fun testPredictTagsForInputStream_TimisoaraBega() {
val image = ImageTagsPredictionTests::class.java.getResource("timisoara-bega.jpg")
val tags = imageTagsPrediction.predictTags(File(image!!.toURI()).inputStream())
assertEquals(
listOf(
"lake",
"nature",
"no people",
"outdoors",
"reflection",
"river",
"sky",
"tranquil",
"tree",
"water"
), tags
)
}
@Test
fun testPredictTagsForInputStream__TimisoaraThrees() {
val image = ImageTagsPredictionTests::class.java.getResource("timisoara-threes.jpg")
val tags = imageTagsPrediction.predictTags(File(image!!.toURI()).inputStream())
assertEquals(
listOf("day", "forest", "growth", "nature", "no people", "outdoors", "plant", "tree"), tags
)
}
@Test
fun testPredictTagsForInputStream_TimisoaraWaterTower() {
val image = ImageTagsPredictionTests::class.java.getResource("timisoara-water-tower.jpg")
val tags = imageTagsPrediction.predictTags(File(image!!.toURI()).inputStream())
assertEquals(
listOf(
"architecture",
"building exterior",
"built structure",
"day",
"history",
"no people",
"outdoors",
"travel destinations"
), tags
)
}
companion object {
private lateinit var imageTagsPrediction: ImageTagsPrediction
@JvmStatic
@BeforeAll
fun setUp() {
imageTagsPrediction = ImageTagsPrediction()
}
@JvmStatic
@AfterAll
fun tearDown() {
imageTagsPrediction.close()
}
}
}

BIN
img-ai/src/test/resources/dev/nuculabs/imagetagger/ai/timisoara-bega.jpg Normal file
View file

Binary file not shown.
Side by side

After

Width:  |  Height:  |  Size: 2.6 MiB

BIN
img-ai/src/test/resources/dev/nuculabs/imagetagger/ai/timisoara-threes.jpg Normal file
View file

Binary file not shown.
Side by side

After

Width:  |  Height:  |  Size: 724 KiB

BIN
img-ai/src/test/resources/dev/nuculabs/imagetagger/ai/timisoara-water-tower.jpg Normal file
View file

Binary file not shown.
Side by side

After

Width:  |  Height:  |  Size: 1.3 MiB

2
img-ui/build.gradle
View file

@ -46,7 +46,7 @@ dependencies {
}
implementation('org.kordamp.ikonli:ikonli-javafx:12.3.1')
implementation('org.kordamp.ikonli:ikonli-fontawesome5-pack:12.3.1')
implementation('com.microsoft.onnxruntime:onnxruntime:1.17.1')
implementation('com.microsoft.onnxruntime:onnxruntime:1.17.1') // delete
implementation("org.jetbrains.kotlinx:kotlinx-coroutines-core:1.8.1-Beta")
testImplementation("org.junit.jupiter:junit-jupiter-api:${junitVersion}")
testRuntimeOnly("org.junit.jupiter:junit-jupiter-engine:${junitVersion}")

7
settings.gradle
View file

@ -1,2 +1,7 @@
plugins {
id 'org.gradle.toolchains.foojay-resolver-convention' version '0.5.0'
}
rootProject.name = "ImageTagger-Solution"
include("img-ui")
include("img-ui")
include 'img-ai'