Running Gemma 4 locally on Mac: Benchmarking Qxotic Jinfer, llama.cpp, MLX, and connecting Java 25 with LangChain4j

java --add-modules jdk.incubator.vector \
     --enable-native-access=ALL-UNNAMED \
     -jar jinfer.jar \
     --model gemma-4-E2B-it-Q4_K_M.gguf \
     --think off -p "Explain quantum computing in three sentences."

git clone --depth 1 https://github.com/ggerganov/llama.cpp
cd llama.cpp
cmake -B build -DGGML_METAL=ON
cmake --build build --config Release -j --target llama-cli

python3 -m venv .venv
source .venv/bin/activate
pip install mlx-lm

import mlx.core as mx
from mlx_lm import load, generate
from mlx_lm.sample_utils import make_sampler

# 1. Wire unified VRAM directly
mx.set_wired_limit(20 * 1024 * 1024 * 1024)

# 2. Load Gemma 4 26B-A4B QAT
model, tokenizer = load('mlx-community/gemma-4-26B-A4B-it-qat-4bit')

prompt = tokenizer.apply_chat_template(
    [{'role': 'user', 'content': 'Explain quantum computing in three sentences.'}],
    tokenize=False, add_generation_prompt=True
)

sampler = make_sampler(temp=0.0)
generate(model, tokenizer, prompt=prompt, max_tokens=2, sampler=sampler) # Warmup

output = generate(model, tokenizer, prompt=prompt, max_tokens=256, sampler=sampler, verbose=True)

mlx_lm.server \
  --model mlx-community/gemma-4-26B-A4B-it-qat-4bit \
  --port 8080 \
  --temp 0.0 \
  --chat-template-args '{"enable_thinking":false}'

<?xml version="1.0" encoding="UTF-8"?>
<project xmlns="http://maven.apache.org/POM/4.0.0"
         xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
    <modelVersion>4.0.0</modelVersion>

    <groupId>com.example.gemma</groupId>
    <artifactId>gemma4-langchain4j-demo</artifactId>
    <version>1.0.0-SNAPSHOT</version>

    <properties>
        <maven.compiler.release>25</maven.compiler.release>
        <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
        <langchain4j.version>1.18.0</langchain4j.version>
    </properties>

    <dependencyManagement>
        <dependencies>
            <dependency>
                <groupId>dev.langchain4j</groupId>
                <artifactId>langchain4j-bom</artifactId>
                <version>${langchain4j.version}</version>
                <type>pom</type>
                <scope>import</scope>
            </dependency>
        </dependencies>
    </dependencyManagement>

    <dependencies>
        <!-- LangChain4j 1.18.0 Core -->
        <dependency>
            <groupId>dev.langchain4j</groupId>
            <artifactId>langchain4j-core</artifactId>
        </dependency>

        <!-- LangChain4j 1.18.0 OpenAI Module -->
        <dependency>
            <groupId>dev.langchain4j</groupId>
            <artifactId>langchain4j-open-ai</artifactId>
        </dependency>

        <!-- SLF4J Logger -->
        <dependency>
            <groupId>org.slf4j</groupId>
            <artifactId>slf4j-simple</artifactId>
            <version>2.0.16</version>
        </dependency>
    </dependencies>

    <build>
        <plugins>
            <plugin>
                <groupId>org.apache.maven.plugins</groupId>
                <artifactId>maven-compiler-plugin</artifactId>
                <version>3.13.0</version>
                <configuration>
                    <release>25</release>
                </configuration>
            </plugin>
            <plugin>
                <groupId>org.codehaus.mojo</groupId>
                <artifactId>exec-maven-plugin</artifactId>
                <version>3.5.0</version>
                <configuration>
                    <mainClass>com.example.gemma.Gemma4LangChain4jApp</mainClass>
                </configuration>
            </plugin>
        </plugins>
    </build>
</project>

package com.example.gemma;

import dev.langchain4j.model.chat.ChatModel;
import dev.langchain4j.model.chat.StreamingChatModel;
import dev.langchain4j.model.openai.OpenAiChatModel;
import dev.langchain4j.model.openai.OpenAiStreamingChatModel;

import java.time.Duration;

import static dev.langchain4j.model.LambdaStreamingResponseHandler.onPartialResponseBlocking;

/**
 * Java 25 application using LangChain4j 1.18.0 with LambdaStreamingResponseHandler
 * static imports to interact with Gemma 4 26B-A4B QAT served locally via MLX OpenAI API server.
 */
public class Gemma4LangChain4jApp {

    public static void main(String[] args) throws Exception {
        String baseUrl = "http://localhost:8080/v1";
        String modelName = "mlx-community/gemma-4-26B-A4B-it-qat-4bit";

        System.out.println("=========================================================");
        System.out.println("  LangChain4j 1.18.0 + Gemma 4 (via MLX OpenAI API)     ");
        System.out.println("=========================================================");
        System.out.println("Connecting to Endpoint : " + baseUrl);
        System.out.println("Model Name             : " + modelName);

        String prompt = "Explain quantum computing in three sentences.";
        System.out.println("\nPrompt: " + prompt);

        // 1. Synchronous ChatModel
        ChatModel model = OpenAiChatModel.builder()
                .baseUrl(baseUrl)
                .modelName(modelName)
                .temperature(0.0)
                .timeout(Duration.ofSeconds(60))
                .build();

        System.out.println("\n--- Synchronous Response ---");
        long startTime = System.currentTimeMillis();
        String responseText = model.chat(prompt);
        long duration = System.currentTimeMillis() - startTime;

        System.out.println(responseText);
        System.out.printf("%n(Completed in %.2f s)%n", duration / 1000.0);

        // 2. Real-Time Streaming ChatModel using static onPartialResponseBlocking
        System.out.println("\n--- Streaming Response ---");
        StreamingChatModel streamingModel = OpenAiStreamingChatModel.builder()
                .baseUrl(baseUrl)
                .modelName(modelName)
                .temperature(0.0)
                .timeout(Duration.ofSeconds(60))
                .build();

        onPartialResponseBlocking(streamingModel, prompt, token -> {
            System.out.print(token);
            System.out.flush();
        });
    }
}

mlx_lm.server \
  --model mlx-community/gemma-4-26B-A4B-it-qat-4bit \
  --port 8080 \
  --temp 0.0 \
  --chat-template-args '{"enable_thinking":false}'

mvn clean compile exec:exec

=========================================================
  LangChain4j 1.18.0 + Gemma 4 (via MLX OpenAI API)     
=========================================================
Connecting to Endpoint : http://localhost:8080/v1
Model Name             : mlx-community/gemma-4-26B-A4B-it-qat-4bit

Prompt: Explain quantum computing in three sentences.

--- Synchronous Response ---
Unlike classical computers that use bits representing either a 0 or 
a 1, quantum computers use qubits that can exist in multiple states 
simultaneously through a phenomenon called superposition. They also 
leverage entanglement, which allows qubits to be deeply interconnected 
so that the state of one instantly influences another, regardless of 
distance. This unique ability allows quantum computers to process vast 
amounts of complex data and solve specific problems much faster than 
even the most powerful traditional supercomputers.

(Completed in 5.43 s)

--- Streaming Response ---
Unlike classical computers that use bits representing either a 0 or
a 1, quantum computers use qubits that can exist in multiple states 
simultaneously through a phenomenon called superposition. They also 
leverage entanglement, which allows qubits to be deeply interconnected 
so that the state of one instantly influences another, regardless of 
distance. This unique ability allows quantum computers to process vast 
amounts of complex data and solve specific problems much faster than 
even the most powerful traditional supercomputers.