Skip to content

Fitness

Jailbreak fitness functions.

JailbreakFitness

Bases: Fitness[LLMAttackGenome]

Fitness function for jailbreak attacks.

Evaluates how successful an attack genome is at bypassing the target LLM's safety measures.

Source code in src/rotalabs_redqueen/llm/fitness.py 
 34
 35
 36
 37
 38
 39
 40
 41
 42
 43
 44
 45
 46
 47
 48
 49
 50
 51
 52
 53
 54
 55
 56
 57
 58
 59
 60
 61
 62
 63
 64
 65
 66
 67
 68
 69
 70
 71
 72
 73
 74
 75
 76
 77
 78
 79
 80
 81
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
class JailbreakFitness(Fitness[LLMAttackGenome]):
    """Fitness function for jailbreak attacks.

    Evaluates how successful an attack genome is at bypassing
    the target LLM's safety measures.
    """

    def __init__(
        self,
        target: LLMTarget,
        judge: Judge | None = None,
        max_retries: int = 3,
        retry_delay: float = 1.0,
    ):
        """Initialize jailbreak fitness.

        Args:
            target: LLM target to attack
            judge: Judge for evaluating responses (default: HeuristicJudge)
            max_retries: Maximum retry attempts on failure
            retry_delay: Delay between retries in seconds
        """
        self.target = target
        self.judge = judge or HeuristicJudge()
        self.max_retries = max_retries
        self.retry_delay = retry_delay

    async def evaluate(self, genome: LLMAttackGenome) -> FitnessResult:
        """Evaluate a single attack genome."""
        prompt = genome.to_prompt()
        error = None
        response_content = ""
        latency_ms = 0.0
        tokens_used = 0

        # Query target with retries
        for attempt in range(self.max_retries):
            try:
                response = await self.target.query(prompt)
                response_content = response.content
                latency_ms = response.latency_ms
                tokens_used = response.tokens_used
                break
            except TargetError as e:
                error = str(e)
                if attempt < self.max_retries - 1:
                    await asyncio.sleep(self.retry_delay)
                continue

        # If all retries failed, return zero fitness
        if error and not response_content:
            return FitnessResult(
                fitness=FitnessValue(0.0),
                behavior=genome.behavior(),
                metadata=EvaluationMetadata(
                    extra={"error": error, "prompt": prompt}
                ),
            )

        # Judge the response
        judge_result = await self.judge.judge(prompt, response_content)

        # Calculate fitness
        # Primary fitness is the judge score (how much it complied)
        # We also factor in relevance
        fitness_value = judge_result.score
        if not judge_result.relevant:
            fitness_value *= 0.5  # Penalize irrelevant responses

        return FitnessResult(
            fitness=FitnessValue(fitness_value),
            behavior=genome.behavior(),
            metadata=EvaluationMetadata(
                extra={
                    "prompt": prompt,
                    "response": response_content[:500],
                    "score": judge_result.score,
                    "refused": judge_result.refused,
                    "harmful_content": judge_result.harmful_content,
                    "relevant": judge_result.relevant,
                    "explanation": judge_result.explanation,
                    "latency_ms": latency_ms,
                    "tokens_used": tokens_used,
                }
            ),
        )

__init__(target: LLMTarget, judge: Judge | None = None, max_retries: int = 3, retry_delay: float = 1.0)

Initialize jailbreak fitness.

Parameters:

Name Type Description Default
target LLMTarget

LLM target to attack

 required
judge Judge | None

Judge for evaluating responses (default: HeuristicJudge)

 None
max_retries int

Maximum retry attempts on failure

 3
retry_delay float

Delay between retries in seconds

 1.0
Source code in src/rotalabs_redqueen/llm/fitness.py 
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
def __init__(
    self,
    target: LLMTarget,
    judge: Judge | None = None,
    max_retries: int = 3,
    retry_delay: float = 1.0,
):
    """Initialize jailbreak fitness.

    Args:
        target: LLM target to attack
        judge: Judge for evaluating responses (default: HeuristicJudge)
        max_retries: Maximum retry attempts on failure
        retry_delay: Delay between retries in seconds
    """
    self.target = target
    self.judge = judge or HeuristicJudge()
    self.max_retries = max_retries
    self.retry_delay = retry_delay

evaluate(genome: LLMAttackGenome) -> FitnessResult async

Evaluate a single attack genome.

Source code in src/rotalabs_redqueen/llm/fitness.py 
 61
 62
 63
 64
 65
 66
 67
 68
 69
 70
 71
 72
 73
 74
 75
 76
 77
 78
 79
 80
 81
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
async def evaluate(self, genome: LLMAttackGenome) -> FitnessResult:
    """Evaluate a single attack genome."""
    prompt = genome.to_prompt()
    error = None
    response_content = ""
    latency_ms = 0.0
    tokens_used = 0

    # Query target with retries
    for attempt in range(self.max_retries):
        try:
            response = await self.target.query(prompt)
            response_content = response.content
            latency_ms = response.latency_ms
            tokens_used = response.tokens_used
            break
        except TargetError as e:
            error = str(e)
            if attempt < self.max_retries - 1:
                await asyncio.sleep(self.retry_delay)
            continue

    # If all retries failed, return zero fitness
    if error and not response_content:
        return FitnessResult(
            fitness=FitnessValue(0.0),
            behavior=genome.behavior(),
            metadata=EvaluationMetadata(
                extra={"error": error, "prompt": prompt}
            ),
        )

    # Judge the response
    judge_result = await self.judge.judge(prompt, response_content)

    # Calculate fitness
    # Primary fitness is the judge score (how much it complied)
    # We also factor in relevance
    fitness_value = judge_result.score
    if not judge_result.relevant:
        fitness_value *= 0.5  # Penalize irrelevant responses

    return FitnessResult(
        fitness=FitnessValue(fitness_value),
        behavior=genome.behavior(),
        metadata=EvaluationMetadata(
            extra={
                "prompt": prompt,
                "response": response_content[:500],
                "score": judge_result.score,
                "refused": judge_result.refused,
                "harmful_content": judge_result.harmful_content,
                "relevant": judge_result.relevant,
                "explanation": judge_result.explanation,
                "latency_ms": latency_ms,
                "tokens_used": tokens_used,
            }
        ),
    )

MultiTargetFitness

Bases: Fitness[LLMAttackGenome]

Fitness that tests against multiple targets.

Useful for finding universal attacks that work across different LLM providers/models.

Source code in src/rotalabs_redqueen/llm/fitness.py 
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
class MultiTargetFitness(Fitness[LLMAttackGenome]):
    """Fitness that tests against multiple targets.

    Useful for finding universal attacks that work across
    different LLM providers/models.
    """

    def __init__(
        self,
        targets: list[LLMTarget],
        judge: Judge | None = None,
        aggregation: str = "mean",
    ):
        """Initialize multi-target fitness.

        Args:
            targets: List of LLM targets to test
            judge: Judge for evaluating responses
            aggregation: How to aggregate scores ("mean", "min", "max")
        """
        self.targets = targets
        self.judge = judge or HeuristicJudge()
        self.aggregation = aggregation
        self._individual_fitness = [
            JailbreakFitness(target, judge) for target in targets
        ]

    async def evaluate(self, genome: LLMAttackGenome) -> FitnessResult:
        """Evaluate genome against all targets."""
        # Evaluate against all targets in parallel
        results = await asyncio.gather(
            *[f.evaluate(genome) for f in self._individual_fitness]
        )

        # Aggregate fitness scores
        scores = [r.fitness.value for r in results]
        if self.aggregation == "mean":
            combined_score = sum(scores) / len(scores)
        elif self.aggregation == "min":
            combined_score = min(scores)
        elif self.aggregation == "max":
            combined_score = max(scores)
        else:
            combined_score = sum(scores) / len(scores)

        # Combine metadata
        target_results = {
            self.targets[i].name: results[i].metadata.extra
            for i in range(len(self.targets))
        }

        return FitnessResult(
            fitness=FitnessValue(combined_score),
            behavior=genome.behavior(),
            metadata=EvaluationMetadata(
                extra={
                    "aggregation": self.aggregation,
                    "individual_scores": scores,
                    "target_results": target_results,
                }
            ),
        )

__init__(targets: list[LLMTarget], judge: Judge | None = None, aggregation: str = 'mean')

Initialize multi-target fitness.

Parameters:

Name Type Description Default
targets list[LLMTarget]

List of LLM targets to test

 required
judge Judge | None

Judge for evaluating responses

 None
aggregation str

How to aggregate scores ("mean", "min", "max")

 'mean'
Source code in src/rotalabs_redqueen/llm/fitness.py 
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
def __init__(
    self,
    targets: list[LLMTarget],
    judge: Judge | None = None,
    aggregation: str = "mean",
):
    """Initialize multi-target fitness.

    Args:
        targets: List of LLM targets to test
        judge: Judge for evaluating responses
        aggregation: How to aggregate scores ("mean", "min", "max")
    """
    self.targets = targets
    self.judge = judge or HeuristicJudge()
    self.aggregation = aggregation
    self._individual_fitness = [
        JailbreakFitness(target, judge) for target in targets
    ]

evaluate(genome: LLMAttackGenome) -> FitnessResult async

Evaluate genome against all targets.

Source code in src/rotalabs_redqueen/llm/fitness.py 
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
async def evaluate(self, genome: LLMAttackGenome) -> FitnessResult:
    """Evaluate genome against all targets."""
    # Evaluate against all targets in parallel
    results = await asyncio.gather(
        *[f.evaluate(genome) for f in self._individual_fitness]
    )

    # Aggregate fitness scores
    scores = [r.fitness.value for r in results]
    if self.aggregation == "mean":
        combined_score = sum(scores) / len(scores)
    elif self.aggregation == "min":
        combined_score = min(scores)
    elif self.aggregation == "max":
        combined_score = max(scores)
    else:
        combined_score = sum(scores) / len(scores)

    # Combine metadata
    target_results = {
        self.targets[i].name: results[i].metadata.extra
        for i in range(len(self.targets))
    }

    return FitnessResult(
        fitness=FitnessValue(combined_score),
        behavior=genome.behavior(),
        metadata=EvaluationMetadata(
            extra={
                "aggregation": self.aggregation,
                "individual_scores": scores,
                "target_results": target_results,
            }
        ),
    )

JailbreakMetrics

Detailed metrics from a jailbreak attempt.

Source code in src/rotalabs_redqueen/llm/fitness.py 
22
23
24
25
26
27
28
29
30
31
@dataclass
class JailbreakMetrics:
    """Detailed metrics from a jailbreak attempt."""

    prompt: str
    response: str
    judge_result: JudgeResult
    latency_ms: float
    tokens_used: int
    error: str | None = None