code:
import { Request } from "express";
import { z } from "zod";
import { config } from "../../../config";
import { OpenAIPromptMessage } from "../../../tokenization";
import { isCompletionRequest } from "../common";
import { RequestPreprocessor } from ".";

const CLAUDE_OUTPUT_MAX = config.maxOutputTokensAnthropic;
const OPENAI_OUTPUT_MAX = config.maxOutputTokensOpenAI;

// https://console.anthropic.com/docs/api/reference#-v1-complete
const AnthropicV1CompleteSchema = z.object({
  model: z.string().regex(/^claude-/, "Model must start with 'claude-'"),
  prompt: z.string({
    required_error:
      "No prompt found. Are you sending an OpenAI-formatted request to the Claude endpoint?",
  }),
  max_tokens_to_sample: z.coerce
    .number()
    .int()
    .transform((v) => Math.min(v, CLAUDE_OUTPUT_MAX)),
  stop_sequences: z.array(z.string()).optional(),
  stream: z.boolean().optional().default(false),
  temperature: z.coerce.number().optional().default(1),
  top_k: z.coerce.number().optional().default(-1),
  top_p: z.coerce.number().optional().default(-1),
  metadata: z.any().optional(),
});

// https://platform.openai.com/docs/api-reference/chat/create
const OpenAIV1ChatCompletionSchema = z.object({
  model: z.string().regex(/^gpt/, "Model must start with 'gpt-'"),
  messages: z.array(
    z.object({
      role: z.enum(["system", "user", "assistant"]),
      content: z.string(),
      name: z.string().optional(),
    }),
    {
      required_error:
        "No prompt found. Are you sending an Anthropic-formatted request to the OpenAI endpoint?",
      invalid_type_error:
        "Messages were not formatted correctly. Refer to the OpenAI Chat API documentation for more information.",
    }
  ),
  temperature: z.number().optional().default(1),
  top_p: z.number().optional().default(1),
  n: z
    .literal(1, {
      errorMap: () => ({
        message: "You may only request a single completion at a time.",
      }),
    })
    .optional(),
  stream: z.boolean().optional().default(false),
  stop: z.union([z.string(), z.array(z.string())]).optional(),
  max_tokens: z.coerce
    .number()
    .int()
    .optional()
    .default(16)
    .transform((v) => Math.min(v, OPENAI_OUTPUT_MAX)),
  frequency_penalty: z.number().optional().default(0),
  presence_penalty: z.number().optional().default(0),
  logit_bias: z.any().optional(),
  user: z.string().optional(),
});

/** Transforms an incoming request body to one that matches the target API. */
export const transformOutboundPayload: RequestPreprocessor = async (req) => {
  const sameService = req.inboundApi === req.outboundApi;
  const alreadyTransformed = req.retryCount > 0;
  const notTransformable = !isCompletionRequest(req);

  if (alreadyTransformed || notTransformable) {
    return;
  }

  if (sameService) {
    const validator =
      req.outboundApi === "openai"
        ? OpenAIV1ChatCompletionSchema
        : AnthropicV1CompleteSchema;
    const result = validator.safeParse(req.body);
    if (!result.success) {
      req.log.error(
        { issues: result.error.issues, body: req.body },
        "Request validation failed"
      );
      throw result.error;
    }
    req.body = result.data;
    return;
  }

  if (req.inboundApi === "openai" && req.outboundApi === "anthropic") {
    req.body = await openaiToAnthropic(req.body, req);
    return;
  }

  throw new Error(
    `'${req.inboundApi}' -> '${req.outboundApi}' request proxying is not supported. Make sure your client is configured to use the correct API.`
  );
};

async function openaiToAnthropic(body: any, req: Request) {
  const result = OpenAIV1ChatCompletionSchema.safeParse(body);
  if (!result.success) {
    req.log.error(
      { issues: result.error.issues, body: req.body },
      "Invalid OpenAI-to-Anthropic request"
    );
    throw result.error;
  }

  // Anthropic has started versioning their API, indicated by an HTTP header
  // `anthropic-version`. The new June 2023 version is not backwards compatible
  // with our OpenAI-to-Anthropic transformations so we need to explicitly
  // request the older version for now. 2023-01-01 will be removed in September.
  // https://docs.anthropic.com/claude/reference/versioning
  req.headers["anthropic-version"] = "2023-01-01";

  const { messages, ...rest } = result.data;
  const prompt = openAIMessagesToClaudePrompt(messages);

  let stops = rest.stop
    ? Array.isArray(rest.stop)
      ? rest.stop
      : [rest.stop]
    : [];
  // Recommended by Anthropic
  stops.push("\n\nHuman:");
  // Helps with jailbreak prompts that send fake system messages and multi-bot
  // chats that prefix bot messages with "System: Respond as <bot name>".
  stops.push("\n\nSystem:");
  // Remove duplicates
  stops = [...new Set(stops)];

  return {
    ...rest,
    // Model may be overridden in `calculate-context-size.ts` to avoid having
    // a circular dependency (`calculate-context-size.ts` needs an already-
    // transformed request body to count tokens, but this function would like
    // to know the count to select a model).
    model: process.env.CLAUDE_SMALL_MODEL || "claude-v1",
    prompt: prompt,
    max_tokens_to_sample: rest.max_tokens,
    stop_sequences: stops,
  };
}

export function openAIMessagesToClaudePrompt(messages: OpenAIPromptMessage[]) {
  return (
    messages
      .map((m) => {
        let role: string = m.role;
        if (role === "assistant") {
          role = "Assistant";
        } else if (role === "system") {
          role = "System";
        } else if (role === "user") {
          role = "Human";
        }
        // https://console.anthropic.com/docs/prompt-design
        // `name` isn't supported by Anthropic but we can still try to use it.
        return `\n\n${role}: ${m.name?.trim() ? `(as ${m.name}) ` : ""}${
          m.content
        }`;
      })
      .join("") + "\n\nAssistant:"
  );
}


code:
import { Request, Response } from "express";
import * as http from "http";
import { buildFakeSseMessage } from "../common";
import { RawResponseBodyHandler, decodeResponseBody } from ".";

type OpenAiChatCompletionResponse = {
  id: string;
  object: string;
  created: number;
  model: string;
  choices: {
    message: { role: string; content: string };
    finish_reason: string | null;
    index: number;
  }[];
};

type AnthropicCompletionResponse = {
  completion: string;
  stop_reason: string;
  truncated: boolean;
  stop: any;
  model: string;
  log_id: string;
  exception: null;
};

/**
 * Consume the SSE stream and forward events to the client. Once the stream is
 * stream is closed, resolve with the full response body so that subsequent
 * middleware can work with it.
 *
 * Typically we would only need of the raw response handlers to execute, but
 * in the event a streamed request results in a non-200 response, we need to
 * fall back to the non-streaming response handler so that the error handler
 * can inspect the error response.
 *
 * Currently most frontends don't support Anthropic streaming, so users can opt
 * to send requests for Claude models via an endpoint that accepts OpenAI-
 * compatible requests and translates the received Anthropic SSE events into
 * OpenAI ones, essentially pretending to be an OpenAI streaming API.
 */
export const handleStreamedResponse: RawResponseBodyHandler = async (
  proxyRes,
  req,
  res
) => {
  // If these differ, the user is using the OpenAI-compatibile endpoint, so
  // we need to translate the SSE events into OpenAI completion events for their
  // frontend.
  if (!req.isStreaming) {
    const err = new Error(
      "handleStreamedResponse called for non-streaming request."
    );
    req.log.error({ stack: err.stack, api: req.inboundApi }, err.message);
    throw err;
  }

  const key = req.key!;
  if (proxyRes.statusCode !== 200) {
    // Ensure we use the non-streaming middleware stack since we won't be
    // getting any events.
    req.isStreaming = false;
    req.log.warn(
      { statusCode: proxyRes.statusCode, key: key.hash },
      `Streaming request returned error status code. Falling back to non-streaming response handler.`
    );
    return decodeResponseBody(proxyRes, req, res);
  }

  return new Promise((resolve, reject) => {
    req.log.info({ key: key.hash }, `Starting to proxy SSE stream.`);

    // Queued streaming requests will already have a connection open and headers
    // sent due to the heartbeat handler.  In that case we can just start
    // streaming the response without sending headers.
    if (!res.headersSent) {
      res.setHeader("Content-Type", "text/event-stream");
      res.setHeader("Cache-Control", "no-cache");
      res.setHeader("Connection", "keep-alive");
      res.setHeader("X-Accel-Buffering", "no");
      copyHeaders(proxyRes, res);
      res.flushHeaders();
    }

    const originalEvents: string[] = [];
    let partialMessage = "";
    let lastPosition = 0;

    type ProxyResHandler<T extends unknown> = (...args: T[]) => void;
    function withErrorHandling<T extends unknown>(fn: ProxyResHandler<T>) {
      return (...args: T[]) => {
        try {
          fn(...args);
        } catch (error) {
          proxyRes.emit("error", error);
        }
      };
    }

    proxyRes.on(
      "data",
      withErrorHandling((chunk: Buffer) => {
        // We may receive multiple (or partial) SSE messages in a single chunk,
        // so we need to buffer and emit seperate stream events for full
        // messages so we can parse/transform them properly.
        const str = chunk.toString();

        // Anthropic uses CRLF line endings (out-of-spec btw)
        const fullMessages = (partialMessage + str).split(/\r?\n\r?\n/);
        partialMessage = fullMessages.pop() || "";

        for (const message of fullMessages) {
          proxyRes.emit("full-sse-event", message);
        }
      })
    );

    proxyRes.on(
      "full-sse-event",
      withErrorHandling((data) => {
        originalEvents.push(data);
        const { event, position } = transformEvent({
          data,
          requestApi: req.inboundApi,
          responseApi: req.outboundApi,
          lastPosition,
        });
        lastPosition = position;
        res.write(event + "\n\n");
      })
    );

    proxyRes.on(
      "end",
      withErrorHandling(() => {
        let finalBody = convertEventsToFinalResponse(originalEvents, req);
        req.log.info({ key: key.hash }, `Finished proxying SSE stream.`);
        res.end();
        resolve(finalBody);
      })
    );

    proxyRes.on("error", (err) => {
      req.log.error({ error: err, key: key.hash }, `Mid-stream error.`);
      const fakeErrorEvent = buildFakeSseMessage(
        "mid-stream-error",
        err.message,
        req
      );
      res.write(`data: ${JSON.stringify(fakeErrorEvent)}\n\n`);
      res.write("data: [DONE]\n\n");
      res.end();
      reject(err);
    });
  });
};

/**
 * Transforms SSE events from the given response API into events compatible with
 * the API requested by the client.
 */
function transformEvent({
  data,
  requestApi,
  responseApi,
  lastPosition,
}: {
  data: string;
  requestApi: string;
  responseApi: string;
  lastPosition: number;
}) {
  if (requestApi === responseApi) {
    return { position: -1, event: data };
  }

  if (requestApi === "anthropic" && responseApi === "openai") {
    throw new Error(`Anthropic -> OpenAI streaming not implemented.`);
  }

  // Anthropic sends the full completion so far with each event whereas OpenAI
  // only sends the delta. To make the SSE events compatible, we remove
  // everything before `lastPosition` from the completion.
  if (!data.startsWith("data:")) {
    return { position: lastPosition, event: data };
  }

  if (data.startsWith("data: [DONE]")) {
    return { position: lastPosition, event: data };
  }

  const event = JSON.parse(data.slice("data: ".length));
  const newEvent = {
    id: "ant-" + event.log_id,
    object: "chat.completion.chunk",
    created: Date.now(),
    model: event.model,
    choices: [
      {
        index: 0,
        delta: { content: event.completion?.slice(lastPosition) },
        finish_reason: event.stop_reason,
      },
    ],
  };
  return {
    position: event.completion.length,
    event: `data: ${JSON.stringify(newEvent)}`,
  };
}

/** Copy headers, excluding ones we're already setting for the SSE response. */
function copyHeaders(proxyRes: http.IncomingMessage, res: Response) {
  const toOmit = [
    "content-length",
    "content-encoding",
    "transfer-encoding",
    "content-type",
    "connection",
    "cache-control",
  ];
  for (const [key, value] of Object.entries(proxyRes.headers)) {
    if (!toOmit.includes(key) && value) {
      res.setHeader(key, value);
    }
  }
}

/**
 * Converts the list of incremental SSE events into an object that resembles a
 * full, non-streamed response from the API so that subsequent middleware can
 * operate on it as if it were a normal response.
 * Events are expected to be in the format they were received from the API.
 */
function convertEventsToFinalResponse(events: string[], req: Request) {
  if (req.outboundApi === "openai") {
    let response: OpenAiChatCompletionResponse = {
      id: "",
      object: "",
      created: 0,
      model: "",
      choices: [],
    };
    response = events.reduce((acc, event, i) => {
      if (!event.startsWith("data: ")) {
        return acc;
      }

      if (event === "data: [DONE]") {
        return acc;
      }

      const data = JSON.parse(event.slice("data: ".length));
      if (i === 0) {
        return {
          id: data.id,
          object: data.object,
          created: data.created,
          model: data.model,
          choices: [
            {
              message: { role: data.choices[0].delta.role, content: "" },
              index: 0,
              finish_reason: null,
            },
          ],
        };
      }

      if (data.choices[0].delta.content) {
        acc.choices[0].message.content += data.choices[0].delta.content;
      }
      acc.choices[0].finish_reason = data.choices[0].finish_reason;
      return acc;
    }, response);
    return response;
  }
  if (req.outboundApi === "anthropic") {
    /*
     * Full complete responses from Anthropic are conveniently just the same as
     * the final SSE event before the "DONE" event, so we can reuse that
     */
    const lastEvent = events[events.length - 2].toString();
    const data = JSON.parse(lastEvent.slice(lastEvent.indexOf("data: ") + "data: ".length));
    const response: AnthropicCompletionResponse = {
      ...data,
      log_id: req.id,
    };
    return response;
  }
  throw new Error("If you get this, something is fucked");
}

repo_name: TheRealZoink/oai

license: unknown


code:
/* This file is fucking horrendous, sorry */
import { Request, Response } from "express";
import * as http from "http";
import util from "util";
import zlib from "zlib";
import { config } from "../../../config";
import { logger } from "../../../logger";
import { keyPool } from "../../../key-management";
import { enqueue, trackWaitTime } from "../../queue";
import { incrementPromptCount } from "../../auth/user-store";
import { isCompletionRequest, writeErrorResponse } from "../common";
import { handleStreamedResponse } from "./handle-streamed-response";
import { logPrompt } from "./log-prompt";

const DECODER_MAP = {
  gzip: util.promisify(zlib.gunzip),
  deflate: util.promisify(zlib.inflate),
  br: util.promisify(zlib.brotliDecompress),
};

const isSupportedContentEncoding = (
  contentEncoding: string
): contentEncoding is keyof typeof DECODER_MAP => {
  return contentEncoding in DECODER_MAP;
};

class RetryableError extends Error {
  constructor(message: string) {
    super(message);
    this.name = "RetryableError";
  }
}

/**
 * Either decodes or streams the entire response body and then passes it as the
 * last argument to the rest of the middleware stack.
 */
export type RawResponseBodyHandler = (
  proxyRes: http.IncomingMessage,
  req: Request,
  res: Response
) => Promise<string | Record<string, any>>;
export type ProxyResHandlerWithBody = (
  proxyRes: http.IncomingMessage,
  req: Request,
  res: Response,
  /**
   * This will be an object if the response content-type is application/json,
   * or if the response is a streaming response. Otherwise it will be a string.
   */
  body: string | Record<string, any>
) => Promise<void>;
export type ProxyResMiddleware = ProxyResHandlerWithBody[];

/**
 * Returns a on.proxyRes handler that executes the given middleware stack after
 * the common proxy response handlers have processed the response and decoded
 * the body.  Custom middleware won't execute if the response is determined to
 * be an error from the upstream service as the response will be taken over by
 * the common error handler.
 *
 * For streaming responses, the handleStream middleware will block remaining
 * middleware from executing as it consumes the stream and forwards events to
 * the client. Once the stream is closed, the finalized body will be attached
 * to res.body and the remaining middleware will execute.
 */
export const createOnProxyResHandler = (apiMiddleware: ProxyResMiddleware) => {
  return async (
    proxyRes: http.IncomingMessage,
    req: Request,
    res: Response
  ) => {
    const initialHandler = req.isStreaming
      ? handleStreamedResponse
      : decodeResponseBody;

    let lastMiddlewareName = initialHandler.name;

    try {
      const body = await initialHandler(proxyRes, req, res);

      const middlewareStack: ProxyResMiddleware = [];

      if (req.isStreaming) {
        // `handleStreamedResponse` writes to the response and ends it, so
        // we can only execute middleware that doesn't write to the response.
        middlewareStack.push(trackRateLimit, incrementKeyUsage, logPrompt);
      } else {
        middlewareStack.push(
          trackRateLimit,
          handleUpstreamErrors,
          incrementKeyUsage,
          copyHttpHeaders,
          logPrompt,
          ...apiMiddleware
        );
      }

      for (const middleware of middlewareStack) {
        lastMiddlewareName = middleware.name;
        await middleware(proxyRes, req, res, body);
      }

      trackWaitTime(req);
    } catch (error: any) {
      // Hack: if the error is a retryable rate-limit error, the request has
      // been re-enqueued and we can just return without doing anything else.
      if (error instanceof RetryableError) {
        return;
      }

      const errorData = {
        error: error.stack,
        thrownBy: lastMiddlewareName,
        key: req.key?.hash,
      };
      const message = `Error while executing proxy response middleware: ${lastMiddlewareName} (${error.message})`;
      if (res.headersSent) {
        req.log.error(errorData, message);
        // This should have already been handled by the error handler, but
        // just in case...
        if (!res.writableEnded) {
          res.end();
        }
        return;
      }
      logger.error(errorData, message);
      res
        .status(500)
        .json({ error: "Internal server error", proxy_note: message });
    }
  };
};

function reenqueueRequest(req: Request) {
  req.log.info(
    { key: req.key?.hash, retryCount: req.retryCount },
    `Re-enqueueing request due to retryable error`
  );
  req.retryCount++;
  enqueue(req);
}

/**
 * Handles the response from the upstream service and decodes the body if
 * necessary.  If the response is JSON, it will be parsed and returned as an
 * object.  Otherwise, it will be returned as a string.
 * @throws {Error} Unsupported content-encoding or invalid application/json body
 */
export const decodeResponseBody: RawResponseBodyHandler = async (
  proxyRes,
  req,
  res
) => {
  if (req.isStreaming) {
    const err = new Error("decodeResponseBody called for a streaming request.");
    req.log.error({ stack: err.stack, api: req.inboundApi }, err.message);
    throw err;
  }

  const promise = new Promise<string>((resolve, reject) => {
    let chunks: Buffer[] = [];
    proxyRes.on("data", (chunk) => chunks.push(chunk));
    proxyRes.on("end", async () => {
      let body = Buffer.concat(chunks);

      const contentEncoding = proxyRes.headers["content-encoding"];
      if (contentEncoding) {
        if (isSupportedContentEncoding(contentEncoding)) {
          const decoder = DECODER_MAP[contentEncoding];
          body = await decoder(body);
        } else {
          const errorMessage = `Proxy received response with unsupported content-encoding: ${contentEncoding}`;
          logger.warn({ contentEncoding, key: req.key?.hash }, errorMessage);
          writeErrorResponse(req, res, 500, {
            error: errorMessage,
            contentEncoding,
          });
          return reject(errorMessage);
        }
      }

      try {
        if (proxyRes.headers["content-type"]?.includes("application/json")) {
          const json = JSON.parse(body.toString());
          return resolve(json);
        }
        return resolve(body.toString());
      } catch (error: any) {
        const errorMessage = `Proxy received response with invalid JSON: ${error.message}`;
        logger.warn({ error, key: req.key?.hash }, errorMessage);
        writeErrorResponse(req, res, 500, { error: errorMessage });
        return reject(errorMessage);
      }
    });
  });
  return promise;
};

// TODO: This is too specific to OpenAI's error responses.
/**
 * Handles non-2xx responses from the upstream service.  If the proxied response
 * is an error, this will respond to the client with an error payload and throw
 * an error to stop the middleware stack.
 * On 429 errors, if request queueing is enabled, the request will be silently
 * re-enqueued.  Otherwise, the request will be rejected with an error payload.
 * @throws {Error} On HTTP error status code from upstream service
 */
const handleUpstreamErrors: ProxyResHandlerWithBody = async (
  proxyRes,
  req,
  res,
  body
) => {
  const statusCode = proxyRes.statusCode || 500;

  if (statusCode < 400) {
    return;
  }

  let errorPayload: Record<string, any>;
  // Subtract 1 from available keys because if this message is being shown,
  // it's because the key is about to be disabled.
  const availableKeys = keyPool.available(req.outboundApi) - 1;
  const tryAgainMessage = Boolean(availableKeys)
    ? `There are ${availableKeys} more keys available; try your request again.`
    : "There are no more keys available.";

  try {
    if (typeof body === "object") {
      errorPayload = body;
    } else {
      throw new Error("Received unparsable error response from upstream.");
    }
  } catch (parseError: any) {
    const statusMessage = proxyRes.statusMessage || "Unknown error";
    // Likely Bad Gateway or Gateway Timeout from reverse proxy/load balancer
    logger.warn(
      { statusCode, statusMessage, key: req.key?.hash },
      parseError.message
    );

    const errorObject = {
      statusCode,
      statusMessage: proxyRes.statusMessage,
      error: parseError.message,
      proxy_note: `This is likely a temporary error with the upstream service.`,
    };
    writeErrorResponse(req, res, statusCode, errorObject);
    throw new Error(parseError.message);
  }

  logger.warn(
    {
      statusCode,
      type: errorPayload.error?.code,
      errorPayload,
      key: req.key?.hash,
    },
    `Received error response from upstream. (${proxyRes.statusMessage})`
  );

  if (statusCode === 400) {
    // Bad request (likely prompt is too long)
    if (req.outboundApi === "openai") {
      errorPayload.proxy_note = `Upstream service rejected the request as invalid. Your prompt may be too long for ${req.body?.model}.`;
    } else if (req.outboundApi === "anthropic") {
      maybeHandleMissingPreambleError(req, errorPayload);
    }
  } else if (statusCode === 401) {
    // Key is invalid or was revoked
    keyPool.disable(req.key!, "revoked");
    errorPayload.proxy_note = `API key is invalid or revoked. ${tryAgainMessage}`;
  } else if (statusCode === 429) {
    // OpenAI uses this for a bunch of different rate-limiting scenarios.
    if (req.outboundApi === "openai") {
      handleOpenAIRateLimitError(req, tryAgainMessage, errorPayload);
    } else if (req.outboundApi === "anthropic") {
      handleAnthropicRateLimitError(req, errorPayload);
    }
  } else if (statusCode === 404) {
    // Most likely model not found
    if (req.outboundApi === "openai") {
      // TODO: this probably doesn't handle GPT-4-32k variants properly if the
      // proxy has keys for both the 8k and 32k context models at the same time.
      if (errorPayload.error?.code === "model_not_found") {
        if (req.key!.isGpt4) {
          errorPayload.proxy_note = `Assigned key isn't provisioned for the GPT-4 snapshot you requested. Try again to get a different key, or use Turbo.`;
        } else {
          errorPayload.proxy_note = `No model was found for this key.`;
        }
      }
    } else if (req.outboundApi === "anthropic") {
      errorPayload.proxy_note = `The requested Claude model might not exist, or the key might not be provisioned for it.`;
    }
  } else {
    errorPayload.proxy_note = `Unrecognized error from upstream service.`;
  }

  // Some OAI errors contain the organization ID, which we don't want to reveal.
  if (errorPayload.error?.message) {
    errorPayload.error.message = errorPayload.error.message.replace(
      /org-.{24}/gm,
      "org-xxxxxxxxxxxxxxxxxxx"
    );
  }

  writeErrorResponse(req, res, statusCode, errorPayload);
  throw new Error(errorPayload.error?.message);
};

/**
 * This is a workaround for a very strange issue where certain API keys seem to
 * enforce more strict input validation than others -- specifically, they will
 * require a `\n\nHuman:` prefix on the prompt, perhaps to prevent the key from
 * being used as a generic text completion service and to enforce the use of
 * the chat RLHF.  This is not documented anywhere, and it's not clear why some
 * keys enforce this and others don't.
 * This middleware checks for that specific error and marks the key as being
 * one that requires the prefix, and then re-enqueues the request.
 * The exact error is:
 * ```
 * {
 *   "error": {
 *     "type": "invalid_request_error",
 *     "message": "prompt must start with \"\n\nHuman:\" turn"
 *   }
 * }
 * ```
 */
function maybeHandleMissingPreambleError(
  req: Request,
  errorPayload: Record<string, any>
) {
  if (
    errorPayload.error?.type === "invalid_request_error" &&
    errorPayload.error?.message === 'prompt must start with "\n\nHuman:" turn'
  ) {
    req.log.warn(
      { key: req.key?.hash },
      "Request failed due to missing preamble. Key will be marked as such for subsequent requests."
    );
    keyPool.update(req.key!, { requiresPreamble: true });
    reenqueueRequest(req);
    throw new RetryableError("Claude request re-enqueued to add preamble.");
  } else {
    errorPayload.proxy_note = `Proxy received unrecognized error from Anthropic. Check the specific error for more information.`;
  }
}

function handleAnthropicRateLimitError(
  req: Request,
  errorPayload: Record<string, any>
) {
  if (errorPayload.error?.type === "rate_limit_error") {
    keyPool.markRateLimited(req.key!);
    reenqueueRequest(req);
    throw new RetryableError("Claude rate-limited request re-enqueued.");
  } else {
    errorPayload.proxy_note = `Unrecognized rate limit error from Anthropic. Key may be over quota.`;
  }
}

function handleOpenAIRateLimitError(
  req: Request,
  tryAgainMessage: string,
  errorPayload: Record<string, any>
): Record<string, any> {
  const type = errorPayload.error?.type;
  if (type === "insufficient_quota") {
    // Billing quota exceeded (key is dead, disable it)
    keyPool.disable(req.key!, "quota");
    errorPayload.proxy_note = `Assigned key's quota has been exceeded. ${tryAgainMessage}`;
  } else if (type === "access_terminated") {
    // Account banned (key is dead, disable it)
    keyPool.disable(req.key!, "revoked");
    errorPayload.proxy_note = `Assigned key has been banned by OpenAI for policy violations. ${tryAgainMessage}`;
  } else if (type === "billing_not_active") {
    // Billing is not active (key is dead, disable it)
    keyPool.disable(req.key!, "revoked");
    errorPayload.proxy_note = `Assigned key was deactivated by OpenAI. ${tryAgainMessage}`;
  } else if (type === "requests" || type === "tokens") {
    // Per-minute request or token rate limit is exceeded, which we can retry
    keyPool.markRateLimited(req.key!);
    // I'm aware this is confusing -- throwing this class of error will cause
    // the proxy response handler to return without terminating the request,
    // so that it can be placed back in the queue.
    reenqueueRequest(req);
    throw new RetryableError("Rate-limited request re-enqueued.");
  } else {
    // OpenAI probably overloaded
    errorPayload.proxy_note = `This is likely a temporary error with OpenAI. Try again in a few seconds.`;
  }
  return errorPayload;
}

const incrementKeyUsage: ProxyResHandlerWithBody = async (_proxyRes, req) => {
  if (isCompletionRequest(req)) {
    keyPool.incrementPrompt(req.key!);
    if (req.user) {
      incrementPromptCount(req.user.token);
    }
  }
};

const trackRateLimit: ProxyResHandlerWithBody = async (proxyRes, req) => {
  keyPool.updateRateLimits(req.key!, proxyRes.headers);
};

const copyHttpHeaders: ProxyResHandlerWithBody = async (
  proxyRes,
  _req,
  res
) => {
  Object.keys(proxyRes.headers).forEach((key) => {
    // Omit content-encoding because we will always decode the response body
    if (key === "content-encoding") {
      return;
    }
    // We're usually using res.json() to send the response, which causes express
    // to set content-length. That's not valid for chunked responses and some
    // clients will reject it so we need to omit it.
    if (key === "transfer-encoding") {
      return;
    }
    res.setHeader(key, proxyRes.headers[key] as string);
  });
};


code:
import { Request } from "express";
import { config } from "../../../config";
import { AIService } from "../../../key-management";
import { logQueue } from "../../../prompt-logging";
import { isCompletionRequest } from "../common";
import { ProxyResHandlerWithBody } from ".";
import { logger } from "../../../logger";

/** If prompt logging is enabled, enqueues the prompt for logging. */
export const logPrompt: ProxyResHandlerWithBody = async (
  _proxyRes,
  req,
  _res,
  responseBody
) => {
  if (!config.promptLogging) {
    return;
  }
  if (typeof responseBody !== "object") {
    throw new Error("Expected body to be an object");
  }

  if (!isCompletionRequest(req)) {
    return;
  }

  const promptPayload = getPromptForRequest(req);
  const promptFlattened = flattenMessages(promptPayload);
  const response = getResponseForService({
    service: req.outboundApi,
    body: responseBody,
  });

  logQueue.enqueue({
    endpoint: req.inboundApi,
    promptRaw: JSON.stringify(promptPayload),
    promptFlattened,
    model: response.model, // may differ from the requested model
    response: response.completion,
  });
};

type OaiMessage = {
  role: "user" | "assistant" | "system";
  content: string;
};

const getPromptForRequest = (req: Request): string | OaiMessage[] => {
  // Since the prompt logger only runs after the request has been proxied, we
  // can assume the body has already been transformed to the target API's
  // format.
  if (req.outboundApi === "anthropic") {
    return req.body.prompt;
  } else {
    return req.body.messages;
  }
};

const flattenMessages = (messages: string | OaiMessage[]): string => {
  if (typeof messages === "string") {
    return messages.trim();
  }
  return messages.map((m) => `${m.role}: ${m.content}`).join("\n");
};

const getResponseForService = ({
  service,
  body,
}: {
  service: AIService;
  body: Record<string, any>;
}): { completion: string; model: string } => {
  if (service === "anthropic") {
    return { completion: body.completion.trim(), model: body.model };
  } else {
    return { completion: body.choices[0].message.content, model: body.model };
  }
};


code:
import { RequestHandler, Request, Router } from "express";
import * as http from "http";
import { createProxyMiddleware } from "http-proxy-middleware";
import { config } from "../config";
import { keyPool } from "../key-management";
import { logger } from "../logger";
import { createQueueMiddleware } from "./queue";
import { ipLimiter } from "./rate-limit";
import { handleProxyError } from "./middleware/common";
import {
  addKey,
  blockZoomerOrigins,
  createPreprocessorMiddleware,
  finalizeBody,
  languageFilter,
  limitCompletions,
  removeOriginHeaders,
} from "./middleware/request";
import {
  createOnProxyResHandler,
  ProxyResHandlerWithBody,
} from "./middleware/response";

let modelsCache: any = null;
let modelsCacheTime = 0;

function getModelsResponse() {
  if (new Date().getTime() - modelsCacheTime < 1000 * 60) {
    return modelsCache;
  }

  // https://platform.openai.com/docs/models/overview
  const gptVariants = [
    "gpt-4",
    "gpt-4-0613",
    "gpt-4-0314", // EOL 2023-09-13
    "gpt-4-32k",
    "gpt-4-32k-0613",
    "gpt-4-32k-0314", // EOL 2023-09-13
    "gpt-3.5-turbo",
    "gpt-3.5-turbo-0301", // EOL 2023-09-13
    "gpt-3.5-turbo-0613",
    "gpt-3.5-turbo-16k",
    "gpt-3.5-turbo-16k-0613",
  ];

  const gpt4Available = keyPool.list().filter((key) => {
    return key.service === "openai" && !key.isDisabled && key.isGpt4;
  }).length;

  const models = gptVariants
    .map((id) => ({
      id,
      object: "model",
      created: new Date().getTime(),
      owned_by: "openai",
      permission: [
        {
          id: "modelperm-" + id,
          object: "model_permission",
          created: new Date().getTime(),
          organization: "*",
          group: null,
          is_blocking: false,
        },
      ],
      root: id,
      parent: null,
    }))
    .filter((model) => {
      if (model.id.startsWith("gpt-4")) {
        return gpt4Available > 0;
      }
      return true;
    });

  modelsCache = { object: "list", data: models };
  modelsCacheTime = new Date().getTime();

  return modelsCache;
}

const handleModelRequest: RequestHandler = (_req, res) => {
  res.status(200).json(getModelsResponse());
};

const rewriteRequest = (
  proxyReq: http.ClientRequest,
  req: Request,
  res: http.ServerResponse
) => {
  const rewriterPipeline = [
    addKey,
    languageFilter,
    limitCompletions,
    blockZoomerOrigins,
    removeOriginHeaders,
    finalizeBody,
  ];

  try {
    for (const rewriter of rewriterPipeline) {
      rewriter(proxyReq, req, res, {});
    }
  } catch (error) {
    req.log.error(error, "Error while executing proxy rewriter");
    proxyReq.destroy(error as Error);
  }
};

const openaiResponseHandler: ProxyResHandlerWithBody = async (
  _proxyRes,
  req,
  res,
  body
) => {
  if (typeof body !== "object") {
    throw new Error("Expected body to be an object");
  }

  if (config.promptLogging) {
    const host = req.get("host");
    body.proxy_note = `Prompts are logged on this proxy instance. See ${host} for more information.`;
  }

  // TODO: Remove once tokenization is stable
  if (req.debug) {
    body.proxy_tokenizer_debug_info = req.debug;
  }

  res.status(200).json(body);
};

const openaiProxy = createQueueMiddleware(
  createProxyMiddleware({
    target: "https://api.openai.com",
    changeOrigin: true,
    on: {
      proxyReq: rewriteRequest,
      proxyRes: createOnProxyResHandler([openaiResponseHandler]),
      error: handleProxyError,
    },
    selfHandleResponse: true,
    logger,
  })
);

const openaiRouter = Router();
// Fix paths because clients don't consistently use the /v1 prefix.
openaiRouter.use((req, _res, next) => {
  if (!req.path.startsWith("/v1/")) {
    req.url = `/v1${req.url}`;
  }
  next();
});
openaiRouter.get("/v1/models", handleModelRequest);
openaiRouter.post(
  "/v1/chat/completions",
  ipLimiter,
  createPreprocessorMiddleware({ inApi: "openai", outApi: "openai" }),
  openaiProxy
);
// Redirect browser requests to the homepage.
openaiRouter.get("*", (req, res, next) => {
  const isBrowser = req.headers["user-agent"]?.includes("Mozilla");
  if (isBrowser) {
    res.redirect("/");
  } else {
    next();
  }
});
openaiRouter.use((req, res) => {
  req.log.warn(`Blocked openai proxy request: ${req.method} ${req.path}`);
  res.status(404).json({ error: "Not found" });
});

export const openai = openaiRouter;


code:
/**
 * Very scuffed request queue. OpenAI's GPT-4 keys have a very strict rate limit
 * of 40000 generated tokens per minute. We don't actually know how many tokens
 * a given key has generated, so our queue will simply retry requests that fail
 * with a non-billing related 429 over and over again until they succeed.
 *
 * Dequeueing can operate in one of two modes:
 * - 'fair': requests are dequeued in the order they were enqueued.
 * - 'random': requests are dequeued randomly, not really a queue at all.
 *
 * When a request to a proxied endpoint is received, we create a closure around
 * the call to http-proxy-middleware and attach it to the request. This allows
 * us to pause the request until we have a key available. Further, if the
 * proxied request encounters a retryable error, we can simply put the request
 * back in the queue and it will be retried later using the same closure.
 */

import type { Handler, Request } from "express";
import { keyPool, SupportedModel } from "../key-management";
import { logger } from "../logger";
import { AGNAI_DOT_CHAT_IP } from "./rate-limit";
import { buildFakeSseMessage } from "./middleware/common";

export type QueuePartition = "claude" | "turbo" | "gpt-4";

const queue: Request[] = [];
const log = logger.child({ module: "request-queue" });

/** Maximum number of queue slots for Agnai.chat requests. */
const AGNAI_CONCURRENCY_LIMIT = 15;
/** Maximum number of queue slots for individual users. */
const USER_CONCURRENCY_LIMIT = 1;

/**
 * Returns a unique identifier for a request. This is used to determine if a
 * request is already in the queue.
 * This can be (in order of preference):
 * - user token assigned by the proxy operator
 * - x-risu-tk header, if the request is from RisuAI.xyz
 * - IP address
 */
function getIdentifier(req: Request) {
  if (req.user) {
    return req.user.token;
  }
  if (req.risuToken) {
    return req.risuToken;
  }
  return req.ip;
}

const sameUserPredicate = (incoming: Request) => (queued: Request) => {
  const queuedId = getIdentifier(queued);
  const incomingId = getIdentifier(incoming);
  return queuedId === incomingId;
};

export function enqueue(req: Request) {
  const enqueuedRequestCount = queue.filter(sameUserPredicate(req)).length;
  let isGuest = req.user?.token === undefined;

  // All Agnai.chat requests come from the same IP, so we allow them to have
  // more spots in the queue. Can't make it unlimited because people will
  // intentionally abuse it.
  // Authenticated users always get a single spot in the queue.
  const maxConcurrentQueuedRequests =
    isGuest && req.ip === AGNAI_DOT_CHAT_IP
      ? AGNAI_CONCURRENCY_LIMIT
      : USER_CONCURRENCY_LIMIT;
  if (enqueuedRequestCount >= maxConcurrentQueuedRequests) {
    if (req.ip === AGNAI_DOT_CHAT_IP) {
      // Re-enqueued requests are not counted towards the limit since they
      // already made it through the queue once.
      if (req.retryCount === 0) {
        throw new Error("Too many agnai.chat requests are already queued");
      }
    } else {
      throw new Error("Your IP or token already has a request in the queue");
    }
  }

  queue.push(req);
  req.queueOutTime = 0;

  // shitty hack to remove hpm's event listeners on retried requests
  removeProxyMiddlewareEventListeners(req);

  // If the request opted into streaming, we need to register a heartbeat
  // handler to keep the connection alive while it waits in the queue. We
  // deregister the handler when the request is dequeued.
  if (req.body.stream === "true" || req.body.stream === true) {
    const res = req.res!;
    if (!res.headersSent) {
      initStreaming(req);
    }
    req.heartbeatInterval = setInterval(() => {
      if (process.env.NODE_ENV === "production") {
        req.res!.write(": queue heartbeat\n\n");
      } else {
        req.log.info(`Sending heartbeat to request in queue.`);
        const partition = getPartitionForRequest(req);
        const avgWait = Math.round(getEstimatedWaitTime(partition) / 1000);
        const currentDuration = Math.round((Date.now() - req.startTime) / 1000);
        const debugMsg = `queue length: ${queue.length}; elapsed time: ${currentDuration}s; avg wait: ${avgWait}s`;
        req.res!.write(buildFakeSseMessage("heartbeat", debugMsg, req));
      }
    }, 10000);
  }

  // Register a handler to remove the request from the queue if the connection
  // is aborted or closed before it is dequeued.
  const removeFromQueue = () => {
    req.log.info(`Removing aborted request from queue.`);
    const index = queue.indexOf(req);
    if (index !== -1) {
      queue.splice(index, 1);
    }
    if (req.heartbeatInterval) {
      clearInterval(req.heartbeatInterval);
    }
  };
  req.onAborted = removeFromQueue;
  req.res!.once("close", removeFromQueue);

  if (req.retryCount ?? 0 > 0) {
    req.log.info({ retries: req.retryCount }, `Enqueued request for retry.`);
  } else {
    req.log.info(`Enqueued new request.`);
  }
}

function getPartitionForRequest(req: Request): QueuePartition {
  // There is a single request queue, but it is partitioned by model and API
  // provider.
  // - claude: requests for the Anthropic API, regardless of model
  // - gpt-4: requests for the OpenAI API, specifically for GPT-4 models
  // - turbo: effectively, all other requests
  const provider = req.outboundApi;
  const model = (req.body.model as SupportedModel) ?? "gpt-3.5-turbo";
  if (provider === "anthropic") {
    return "claude";
  }
  if (provider === "openai" && model.startsWith("gpt-4")) {
    return "gpt-4";
  }
  return "turbo";
}

function getQueueForPartition(partition: QueuePartition): Request[] {
  return queue.filter((req) => getPartitionForRequest(req) === partition);
}

export function dequeue(partition: QueuePartition): Request | undefined {
  const modelQueue = getQueueForPartition(partition);

  if (modelQueue.length === 0) {
    return undefined;
  }

  const req = modelQueue.reduce((prev, curr) =>
    prev.startTime < curr.startTime ? prev : curr
  );
  queue.splice(queue.indexOf(req), 1);

  if (req.onAborted) {
    req.res!.off("close", req.onAborted);
    req.onAborted = undefined;
  }

  if (req.heartbeatInterval) {
    clearInterval(req.heartbeatInterval);
  }

  // Track the time leaving the queue now, but don't add it to the wait times
  // yet because we don't know if the request will succeed or fail. We track
  // the time now and not after the request succeeds because we don't want to
  // include the model processing time.
  req.queueOutTime = Date.now();
  return req;
}

/**
 * Naive way to keep the queue moving by continuously dequeuing requests. Not
 * ideal because it limits throughput but we probably won't have enough traffic
 * or keys for this to be a problem.  If it does we can dequeue multiple
 * per tick.
 **/
function processQueue() {
  // This isn't completely correct, because a key can service multiple models.
  // Currently if a key is locked out on one model it will also stop servicing
  // the others, because we only track one rate limit per key.
  const gpt4Lockout = keyPool.getLockoutPeriod("gpt-4");
  const turboLockout = keyPool.getLockoutPeriod("gpt-3.5-turbo");
  const claudeLockout = keyPool.getLockoutPeriod("claude-v1");

  const reqs: (Request | undefined)[] = [];
  if (gpt4Lockout === 0) {
    reqs.push(dequeue("gpt-4"));
  }
  if (turboLockout === 0) {
    reqs.push(dequeue("turbo"));
  }
  if (claudeLockout === 0) {
    reqs.push(dequeue("claude"));
  }

  reqs.filter(Boolean).forEach((req) => {
    if (req?.proceed) {
      req.log.info({ retries: req.retryCount }, `Dequeuing request.`);
      req.proceed();
    }
  });
  setTimeout(processQueue, 50);
}

/**
 * Kill stalled requests after 5 minutes, and remove tracked wait times after 2
 * minutes.
 **/
function cleanQueue() {
  const now = Date.now();
  const oldRequests = queue.filter(
    (req) => now - (req.startTime ?? now) > 5 * 60 * 1000
  );
  oldRequests.forEach((req) => {
    req.log.info(`Removing request from queue after 5 minutes.`);
    killQueuedRequest(req);
  });

  const index = waitTimes.findIndex(
    (waitTime) => now - waitTime.end > 300 * 1000
  );
  const removed = waitTimes.splice(0, index + 1);
  log.trace(
    { stalledRequests: oldRequests.length, prunedWaitTimes: removed.length },
    `Cleaning up request queue.`
  );
  setTimeout(cleanQueue, 20 * 1000);
}

export function start() {
  processQueue();
  cleanQueue();
  log.info(`Started request queue.`);
}

let waitTimes: { partition: QueuePartition; start: number; end: number }[] = [];

/** Adds a successful request to the list of wait times. */
export function trackWaitTime(req: Request) {
  waitTimes.push({
    partition: getPartitionForRequest(req),
    start: req.startTime!,
    end: req.queueOutTime ?? Date.now(),
  });
}

/** Returns average wait time in milliseconds. */
export function getEstimatedWaitTime(partition: QueuePartition) {
  const now = Date.now();
  const recentWaits = waitTimes.filter(
    (wt) => wt.partition === partition && now - wt.end < 300 * 1000
  );
  if (recentWaits.length === 0) {
    return 0;
  }

  return (
    recentWaits.reduce((sum, wt) => sum + wt.end - wt.start, 0) /
    recentWaits.length
  );
}

export function getQueueLength(partition: QueuePartition | "all" = "all") {
  if (partition === "all") {
    return queue.length;
  }
  const modelQueue = getQueueForPartition(partition);
  return modelQueue.length;
}

export function createQueueMiddleware(proxyMiddleware: Handler): Handler {
  return (req, res, next) => {
    req.proceed = () => {
      proxyMiddleware(req, res, next);
    };

    try {
      enqueue(req);
    } catch (err: any) {
      req.res!.status(429).json({
        type: "proxy_error",
        message: err.message,
        stack: err.stack,
        proxy_note: `Only one request can be queued at a time. If you don't have another request queued, your IP or user token might be in use by another request.`,
      });
    }
  };
}

function killQueuedRequest(req: Request) {
  if (!req.res || req.res.writableEnded) {
    req.log.warn(`Attempted to terminate request that has already ended.`);
    return;
  }
  const res = req.res;
  try {
    const message = `Your request has been terminated by the proxy because it has been in the queue for more than 5 minutes. The queue is currently ${queue.length} requests long.`;
    if (res.headersSent) {
      const fakeErrorEvent = buildFakeSseMessage(
        "proxy queue error",
        message,
        req
      );
      res.write(fakeErrorEvent);
      res.end();
    } else {
      res.status(500).json({ error: message });
    }
  } catch (e) {
    req.log.error(e, `Error killing stalled request.`);
  }
}

function initStreaming(req: Request) {
  req.log.info(`Initiating streaming for new queued request.`);
  const res = req.res!;
  res.statusCode = 200;
  res.setHeader("Content-Type", "text/event-stream");
  res.setHeader("Cache-Control", "no-cache");
  res.setHeader("Connection", "keep-alive");
  res.setHeader("X-Accel-Buffering", "no"); // nginx-specific fix
  res.flushHeaders();
  res.write("\n");
  res.write(": joining queue\n\n");
}

/**
 * http-proxy-middleware attaches a bunch of event listeners to the req and
 * res objects which causes problems with our approach to re-enqueuing failed
 * proxied requests. This function removes those event listeners.
 * We don't have references to the original event listeners, so we have to
 * look through the list and remove HPM's listeners by looking for particular
 * strings in the listener functions. This is an astoundingly shitty way to do
 * this, but it's the best I can come up with.
 */
function removeProxyMiddlewareEventListeners(req: Request) {
  // node_modules/http-proxy-middleware/dist/plugins/default/debug-proxy-errors-plugin.js:29
  // res.listeners('close')
  const RES_ONCLOSE = `Destroying proxyRes in proxyRes close event`;
  // node_modules/http-proxy-middleware/dist/plugins/default/debug-proxy-errors-plugin.js:19
  // res.listeners('error')
  const RES_ONERROR = `Socket error in proxyReq event`;
  // node_modules/http-proxy/lib/http-proxy/passes/web-incoming.js:146
  // req.listeners('aborted')
  const REQ_ONABORTED = `proxyReq.abort()`;
  // node_modules/http-proxy/lib/http-proxy/passes/web-incoming.js:156
  // req.listeners('error')
  const REQ_ONERROR = `if (req.socket.destroyed`;

  const res = req.res!;

  const resOnClose = res
    .listeners("close")
    .find((listener) => listener.toString().includes(RES_ONCLOSE));
  if (resOnClose) {
    res.removeListener("close", resOnClose as any);
  }

  const resOnError = res
    .listeners("error")
    .find((listener) => listener.toString().includes(RES_ONERROR));
  if (resOnError) {
    res.removeListener("error", resOnError as any);
  }

  const reqOnAborted = req
    .listeners("aborted")
    .find((listener) => listener.toString().includes(REQ_ONABORTED));
  if (reqOnAborted) {
    req.removeListener("aborted", reqOnAborted as any);
  }

  const reqOnError = req
    .listeners("error")
    .find((listener) => listener.toString().includes(REQ_ONERROR));
  if (reqOnError) {
    req.removeListener("error", reqOnError as any);
  }
}


code:
import { Request, Response, NextFunction } from "express";
import { config } from "../config";

export const AGNAI_DOT_CHAT_IP = "157.230.249.32";

const RATE_LIMIT_ENABLED = Boolean(config.modelRateLimit);
const RATE_LIMIT = Math.max(1, config.modelRateLimit);
const ONE_MINUTE_MS = 60 * 1000;

const lastAttempts = new Map<string, number[]>();

const expireOldAttempts = (now: number) => (attempt: number) =>
  attempt > now - ONE_MINUTE_MS;

const getTryAgainInMs = (ip: string) => {
  const now = Date.now();
  const attempts = lastAttempts.get(ip) || [];
  const validAttempts = attempts.filter(expireOldAttempts(now));

  if (validAttempts.length >= RATE_LIMIT) {
    return validAttempts[0] - now + ONE_MINUTE_MS;
  } else {
    lastAttempts.set(ip, [...validAttempts, now]);
    return 0;
  }
};

const getStatus = (ip: string) => {
  const now = Date.now();
  const attempts = lastAttempts.get(ip) || [];
  const validAttempts = attempts.filter(expireOldAttempts(now));
  return {
    remaining: Math.max(0, RATE_LIMIT - validAttempts.length),
    reset: validAttempts.length > 0 ? validAttempts[0] + ONE_MINUTE_MS : now,
  };
};

/** Prunes attempts and IPs that are no longer relevant after one minutes. */
const clearOldAttempts = () => {
  const now = Date.now();
  for (const [ip, attempts] of lastAttempts.entries()) {
    const validAttempts = attempts.filter(expireOldAttempts(now));
    if (validAttempts.length === 0) {
      lastAttempts.delete(ip);
    } else {
      lastAttempts.set(ip, validAttempts);
    }
  }
};
setInterval(clearOldAttempts, 10 * 1000);

export const getUniqueIps = () => {
  return lastAttempts.size;
};

export const ipLimiter = async (
  req: Request,
  res: Response,
  next: NextFunction
) => {
  if (!RATE_LIMIT_ENABLED) {
    next();
    return;
  }

  // Exempt Agnai.chat from rate limiting since it's shared between a lot of
  // users. Dunno how to prevent this from being abused without some sort of
  // identifier sent from Agnaistic to identify specific users.
  if (req.ip === AGNAI_DOT_CHAT_IP) {
    next();
    return;
  }

  // If user is authenticated, key rate limiting by their token. Otherwise, key
  // rate limiting by their IP address. Mitigates key sharing.
  const rateLimitKey = req.user?.token || req.risuToken || req.ip;

  const { remaining, reset } = getStatus(rateLimitKey);
  res.set("X-RateLimit-Limit", config.modelRateLimit.toString());
  res.set("X-RateLimit-Remaining", remaining.toString());
  res.set("X-RateLimit-Reset", reset.toString());

  const tryAgainInMs = getTryAgainInMs(rateLimitKey);
  if (tryAgainInMs > 0) {
    res.set("Retry-After", tryAgainInMs.toString());
    res.status(429).json({
      error: {
        type: "proxy_rate_limited",
        message: `This proxy is rate limited to ${
          config.modelRateLimit
        } prompts per minute. Please try again in ${Math.ceil(
          tryAgainInMs / 1000
        )} seconds.`,
      },
    });
  } else {
    next();
  }
};


code:
/* Accepts incoming requests at either the /kobold or /openai routes and then
routes them to the appropriate handler to be forwarded to the OpenAI API.
Incoming OpenAI requests are more or less 1:1 with the OpenAI API, but only a
subset of the API is supported. Kobold requests must be transformed into
equivalent OpenAI requests. */

import * as express from "express";
import { gatekeeper } from "./auth/gatekeeper";
import { checkRisuToken } from "./auth/check-risu-token";
import { kobold } from "./kobold";
import { openai } from "./openai";
import { anthropic } from "./anthropic";

const proxyRouter = express.Router();
proxyRouter.use(
  express.json({ limit: "1536kb" }),
  express.urlencoded({ extended: true, limit: "1536kb" })
);
proxyRouter.use(gatekeeper);
proxyRouter.use(checkRisuToken);
proxyRouter.use((req, _res, next) => {
  req.startTime = Date.now();
  req.retryCount = 0;
  next();
});
proxyRouter.use("/kobold", kobold);
proxyRouter.use("/openai", openai);
proxyRouter.use("/anthropic", anthropic);
export { proxyRouter as proxyRouter };


code:
import { assertConfigIsValid, config } from "./config";
import "source-map-support/register";
import express from "express";
import cors from "cors";
import path from "path";
import pinoHttp from "pino-http";
import childProcess from "child_process";
import { logger } from "./logger";
import { keyPool } from "./key-management";
import { adminRouter } from "./admin/routes";
import { proxyRouter } from "./proxy/routes";
import { handleInfoPage, handleStatusPage } from "./info-page";
import { logQueue } from "./prompt-logging";
import { start as startRequestQueue } from "./proxy/queue";
import { init as initUserStore } from "./proxy/auth/user-store";
import { init as initTokenizers } from "./tokenization";
import { checkOrigin } from "./proxy/check-origin";

import * as userStore from "./proxy/auth/user-store";
import { UserSchema, UserSchemaWithToken, parseSort, sortBy } from "./admin/common";

const PORT = config.port;

const app = express();
// middleware
app.use(
  pinoHttp({
    quietReqLogger: true,
    logger,
    autoLogging: {
      ignore: (req) => {
        const ignored = ["/proxy/kobold/api/v1/model", "/health", "/users-stats"];
        return ignored.includes(req.url as string);
      },
    },
    redact: {
      paths: [
        "req.headers.cookie",
        'res.headers["set-cookie"]',
        "req.headers.authorization",
        'req.headers["x-api-key"]',
        // Don't log the prompt text on transform errors
        "body.messages",
        "body.prompt",
      ],
      censor: "********",
    },
  })
);

// TODO: Detect (or support manual configuration of) whether the app is behind
// a load balancer/reverse proxy, which is necessary to determine request IP
// addresses correctly.
app.set("trust proxy", true);

app.set("view engine", "ejs");
app.set("views", path.join(__dirname, "views"));

app.get("/health", (_req, res) => res.sendStatus(200));

app.get("/status", (_req, res) => res.send(handleStatusPage(_req)));

function get_user_stats() {
	const sort = ["promptCount", "lastUsedAt"];
    const users = userStore.getPublicUsers();
    return { users, count: users.length };
}

app.get("/users-stats", (_req, res) => res.send(get_user_stats())

);

app.use(cors());
app.use(checkOrigin);

// routes
app.get("/", handleInfoPage);
app.use("/admin", adminRouter);
app.use("/proxy", proxyRouter);

// 500 and 404
app.use((err: any, _req: unknown, res: express.Response, _next: unknown) => {
  if (err.status) {
    res.status(err.status).json({ error: err.message });
  } else {
    logger.error(err);
    res.status(500).json({
      error: {
        type: "proxy_error",
        message: err.message,
        stack: err.stack,
        proxy_note: `Reverse proxy encountered an internal server error.`,
      },
    });
  }
});
app.use((_req: unknown, res: express.Response) => {
  res.status(404).json({ error: "Not found" });
});

async function start() {
  logger.info("Server starting up...");
  await setBuildInfo();

  logger.info("Checking configs and external dependencies...");
  await assertConfigIsValid();

  keyPool.init();

  await initTokenizers();

  if (config.gatekeeper === "user_token") {
    await initUserStore();
  }

  if (config.promptLogging) {
    logger.info("Starting prompt logging...");
    logQueue.start();
  }

  logger.info("Starting request queue...");
  startRequestQueue();

  app.listen(PORT, async () => {
    logger.info({ port: PORT }, "Now listening for connections.");
    registerUncaughtExceptionHandler();
  });

  logger.info(
    { build: process.env.BUILD_INFO, nodeEnv: process.env.NODE_ENV },
    "Startup complete."
  );
}

function registerUncaughtExceptionHandler() {
  process.on("uncaughtException", (err: any) => {
    logger.error(
      { err, stack: err?.stack },
      "UNCAUGHT EXCEPTION. Please report this error trace."
    );
  });
  process.on("unhandledRejection", (err: any) => {
    logger.error(
      { err, stack: err?.stack },
      "UNCAUGHT PROMISE REJECTION. Please report this error trace."
    );
  });
}

/**
 * Attepts to collect information about the current build from either the
 * environment or the git repo used to build the image (only works if not
 * .dockerignore'd). If you're running a sekrit club fork, you can no-op this
 * function and set the BUILD_INFO env var manually, though I would prefer you
 * didn't set it to something misleading.
 */
async function setBuildInfo() {
  // Render .dockerignore's the .git directory but provides info in the env
  if (process.env.RENDER) {
    const sha = process.env.RENDER_GIT_COMMIT?.slice(0, 7) || "unknown SHA";
    const branch = process.env.RENDER_GIT_BRANCH || "unknown branch";
    const repo = process.env.RENDER_GIT_REPO_SLUG || "unknown repo";
    const buildInfo = `${sha} (${branch}@${repo})`;
    process.env.BUILD_INFO = buildInfo;
    logger.info({ build: buildInfo }, "Got build info from Render config.");
    return;
  }

  try {
    // Ignore git's complaints about dubious directory ownership on Huggingface
    // (which evidently runs dockerized Spaces on Windows with weird NTFS perms)
    if (process.env.SPACE_ID) {
      childProcess.execSync("git config --global --add safe.directory /app");
    }

    const promisifyExec = (cmd: string) =>
      new Promise((resolve, reject) => {
        childProcess.exec(cmd, (err, stdout) =>
          err ? reject(err) : resolve(stdout)
        );
      });

    const promises = [
      promisifyExec("git rev-parse --short HEAD"),
      promisifyExec("git rev-parse --abbrev-ref HEAD"),
      promisifyExec("git config --get remote.origin.url"),
      promisifyExec("git status --porcelain"),
    ].map((p) => p.then((result: any) => result.toString().trim()));

    let [sha, branch, remote, status] = await Promise.all(promises);

    remote = remote.match(/.*[\/:]([\w-]+)\/([\w\-\.]+?)(?:\.git)?$/) || [];
    const repo = remote.slice(-2).join("/");
    status = status
      // ignore Dockerfile changes since that's how the user deploys the app
      .split("\n")
      .filter((line: string) => !line.endsWith("Dockerfile") && line);

    const changes = status.length > 0;

    const build = `${sha}${changes ? " (modified)" : ""} (${branch}@${repo})`;
    process.env.BUILD_INFO = build;
    logger.info({ build, status, changes }, "Got build info from Git.");
  } catch (error: any) {
    logger.error(
      {
        error,
        stdout: error.stdout?.toString(),
        stderr: error.stderr?.toString(),
      },
      "Failed to get commit SHA.",
      error
    );
    process.env.BUILD_INFO = "unknown";
  }
}

start();


code:
import { getTokenizer } from "@anthropic-ai/tokenizer";
import { Tiktoken } from "tiktoken/lite";

let encoder: Tiktoken;

export function init() {
  // they export a `countTokens` function too but it instantiates a new
  // tokenizer every single time and it is not fast...
  encoder = getTokenizer();
  return true;
}

export function getTokenCount(prompt: string, _model: string) {
  // Don't try tokenizing if the prompt is massive to prevent DoS.
  // 500k characters should be sufficient for all supported models.
  if (prompt.length > 500000) {
    return {
      tokenizer: "length fallback",
      token_count: 100000,
    };
  }

  return {
    tokenizer: "@anthropic-ai/tokenizer",
    token_count: encoder.encode(prompt.normalize("NFKC"), "all").length,
  };
}


code:
export { OpenAIPromptMessage } from "./openai";
export { init, countTokens } from "./tokenizer";


code:
import { Tiktoken } from "tiktoken/lite";
import cl100k_base from "tiktoken/encoders/cl100k_base.json";

let encoder: Tiktoken;

export function init() {
  encoder = new Tiktoken(
    cl100k_base.bpe_ranks,
    cl100k_base.special_tokens,
    cl100k_base.pat_str
  );
  return true;
}

// Tested against:
// https://github.com/openai/openai-cookbook/blob/main/examples/How_to_count_tokens_with_tiktoken.ipynb

export function getTokenCount(messages: any[], model: string) {
  const gpt4 = model.startsWith("gpt-4");

  const tokensPerMessage = gpt4 ? 3 : 4;
  const tokensPerName = gpt4 ? 1 : -1; // turbo omits role if name is present

  let numTokens = 0;

  for (const message of messages) {
    numTokens += tokensPerMessage;
    for (const key of Object.keys(message)) {
      {
        const value = message[key];
        // Break if we get a huge message or exceed the token limit to prevent
        // DoS.
        // 100k tokens allows for future 100k GPT-4 models and 500k characters
        // is just a sanity check
        if (value.length > 500000 || numTokens > 100000) {
          numTokens = 100000;
          return {
            tokenizer: "tiktoken (prompt length limit exceeded)",
            token_count: numTokens,
          };
        }

        numTokens += encoder.encode(message[key]).length;
        if (key === "name") {
          numTokens += tokensPerName;
        }
      }
    }
  }
  numTokens += 3; // every reply is primed with <|start|>assistant<|message|>
  return { tokenizer: "tiktoken", token_count: numTokens };
}

export type OpenAIPromptMessage = {
  name?: string;
  content: string;
  role: string;
};


code:
import { Request } from "express";
import { config } from "../config";
import {
  init as initClaude,
  getTokenCount as getClaudeTokenCount,
} from "./claude";
import {
  init as initOpenAi,
  getTokenCount as getOpenAITokenCount,
  OpenAIPromptMessage,
} from "./openai";

export async function init() {
  if (config.anthropicKey) {
    initClaude();
  }
  if (config.openaiKey) {
    initOpenAi();
  }
}

type TokenCountResult = {
  token_count: number;
  tokenizer: string;
  tokenization_duration_ms: number;
};
type TokenCountRequest = {
  req: Request;
} & (
  | { prompt: string; service: "anthropic" }
  | { prompt: OpenAIPromptMessage[]; service: "openai" }
);
export async function countTokens({
  req,
  service,
  prompt,
}: TokenCountRequest): Promise<TokenCountResult> {
  const time = process.hrtime();
  switch (service) {
    case "anthropic":
      return {
        ...getClaudeTokenCount(prompt, req.body.model),
        tokenization_duration_ms: getElapsedMs(time),
      };
    case "openai":
      return {
        ...getOpenAITokenCount(prompt, req.body.model),
        tokenization_duration_ms: getElapsedMs(time),
      };
    default:
      throw new Error(`Unknown service: ${service}`);
  }
}

function getElapsedMs(time: [number, number]) {
  const diff = process.hrtime(time);
  return diff[0] * 1000 + diff[1] / 1e6;
}


code:
import { Express } from "express-serve-static-core";
import { AIService, Key } from "../key-management/index";
import { User } from "../proxy/auth/user-store";

declare global {
  namespace Express {
    interface Request {
      key?: Key;
      /** Denotes the format of the user's submitted request. */
      inboundApi: AIService | "kobold";
      /** Denotes the format of the request being proxied to the API. */
      outboundApi: AIService;
      /** If the request comes from a RisuAI.xyz user, this is their token. */
      risuToken?: string;
      user?: User;
      isStreaming?: boolean;
      startTime: number;
      retryCount: number;
      queueOutTime?: number;
      onAborted?: () => void;
      proceed: () => void;
      heartbeatInterval?: NodeJS.Timeout;
      promptTokens?: number;
      outputTokens?: number;
      // TODO: remove later
      debug: Record<string, any>;
    }
  }
}


code:
<!DOCTYPE html>
<html>
  <head>
    <meta charset="utf-8">
    <meta name="csrf-token" content="<%= csrfToken %>">
    <title><%= title %></title>
    <style>
      .pagination {
        list-style-type: none;
        padding: 0;
      }
      .pagination li {
        display: inline-block;
      }
      .pagination li a {
        display: block;
        padding: 0.5em 1em;
        text-decoration: none;
      }
      .pagination li.active a {
        background-color: #58739c;
        color: #fff;
      }

      table {
        border-collapse: collapse;
        border: 1px solid #ccc;
      }
      table td, table th {
        border: 1px solid #ccc;
        padding: 0.25em 0.5em;
      }
      th.active {
        background-color: #e0e6f6;
      }
      td.actions {
        padding: 0;
        text-align: center;
      }
      td.actions a {
        text-decoration: none;
        padding: 0.5em;
        height: 100%;
        width: 100%;
      }
      td.actions:hover {
        background-color: #ccc;
      }

      @media (max-width: 600px) {
        table {
          width: 100%;
        }
        table td, table th {
          display: block;
          width: 100%;
        }
      }
    </style>
  </head>
  <body style="font-family: sans-serif; background-color: #f0f0f0; padding: 1em;">


code:
  Page Size
  <select id="pageSize" onchange="setPageSize(this.value)" style="margin-bottom: 1rem;">
    <option value="10" <% if (pageSize === 10) { %>selected<% } %>>10</option>
    <option value="20" <% if (pageSize === 20) { %>selected<% } %>>20</option>
    <option value="50" <% if (pageSize === 50) { %>selected<% } %>>50</option>
    <option value="100" <% if (pageSize === 100) { %>selected<% } %>>100</option>
    <option value="200" <% if (pageSize === 200) { %>selected<% } %>>200</option>
  </select>


code:
<%- include("../_partials/admin-header", { title: "Create User - OAI Reverse Proxy Admin" }) %>

Create Permament User Token

<form action="/admin/manage/create-user" method="post">
  <input type="hidden" name="_csrf" value="<%= csrfToken %>" />
  <input type="submit" value="Create" />
</form>

Create Prompt Lmited Temporary User Token

<form action="/admin/manage/create-temp-user" method="post">
  <input type="hidden" name="_csrf" value="<%= csrfToken %>" />
  <input type="hidden" name="timeLimit" value="-1" />
  <input type="submit" value="Create" /> Prompt limit: <input type="number" value="100" name="promptLimit" />
</form>

Create Time Lmited Temporary User Token

<form action="/admin/manage/create-temp-user" method="post">
  <input type="hidden" name="_csrf" value="<%= csrfToken %>" />
  <input type="hidden" name="promptLimit" value="-1" />
  <input type="submit" value="Create" /> Time limit (Seconds): <input type="number" value="3600" name="timeLimit" />

 1 hour   = 3600

 6 hours  = 21600

 12 hours = 43200

 24 hours = 86400

</form>
<% if (newToken) { %>

Just created <%= recentUsers[0].token %>.

<% } %>

Recent Tokens

  <% recentUsers.forEach(function(user) { %>

<a href="/admin/manage/view-user/<%= user.token %>"><%= user.token %>

  <% }) %>

<%- include("../_partials/admin-footer") %>


code:
<%- include("../_partials/admin-header", { title: "Export Users - OAI Reverse Proxy Admin" }) %>

Export Users

  Export users to JSON. The JSON will be an array of objects under the key
  users. You can use this JSON to import users later.

<button onclick="exportUsers()">Export</button>
<%- include("../_partials/admin-footer") %>


code:
<%- include("../_partials/admin-header", { title: "Import Users - OAI Reverse Proxy Admin" }) %>

Import Users

  Import users from JSON. The JSON should be an array of objects under the key
  users. Each object should have the following fields:

token (required): a unique identifier for the user

ip (optional): IP addresses the user has connected from

    type (optional): either normal or
    special

    promptCount (optional): the number of times the user has sent a
    prompt

    tokenCount (optional): the number of tokens the user has
    consumed (not yet implemented)

    createdAt (optional): the timestamp when the user was created

    disabledAt (optional): the timestamp when the user was disabled

    disabledReason (optional): the reason the user was disabled

  If a user with the same token already exists, the existing user will be
  updated with the new values.

<form action="/admin/manage/import-users?_csrf=<%= csrfToken %>" method="post" enctype="multipart/form-data">
  <input type="file" name="users" />
  <input type="submit" value="Import" />
</form>
</form>
<% if (imported > 0) { %>

Imported <%= imported %> users.

<% } %>
<%- include("../_partials/admin-footer") %>


code:
<%- include("../_partials/admin-header", { title: "OAI Reverse Proxy Admin" }) %>

OAI Reverse Proxy Admin

<% if (!isPersistenceEnabled) { %>

  ⚠ Users will be lost when the server restarts because persistence is
  not configured.

Be sure to export your users and import them again after restarting the
  server if you want to keep them.

 See the
  user management documentation to learn how to set up persistence.

<% } %>

List Users

Create User

Import Users

Export Users

<%- include("../_partials/admin-footer") %>


code:
<%- include("../_partials/admin-header", { title: "Users - OAI Reverse Proxy Admin" }) %>

User Token List

    <input type="hidden" name="_csrf" value="<%= csrfToken %>" />

    <% if (users.length === 0) { %>

No users found.

    <% } else { %>

          Token
          <th <% if (sort.includes("ip")) { %>class="active"<% } %> >IPs
          <th <% if (sort.includes("promptCount")) { %>class="active"<% } %> >Prompts
          Type
          <th <% if (sort.includes("createdAt")) { %>class="active"<% } %> >Created (UTC)
          <th <% if (sort.includes("lastUsedAt")) { %>class="active"<% } %> >Last Used (UTC)
          Banned?

        <% users.forEach(function(user){ %>

            <a href="/admin/manage/view-user/<%= user.token %>"><%= user.token %>

          <%= user.ip.length %>
          <%= user.promptCount %>
          <%= user.type %>
          <%= user.createdAt %>
          <%= user.lastUsedAt ?? "never" %>

            <% if (user.disabledAt) { %>
            <a title="Unban" href="#" class="unban" data-token="<%= user.token %>">🔄
            <% } else { %>
            <a title="Ban" href="#" class="ban" data-token="<%= user.token %>">🚫
            <% } %>
          <%= user.disabledAt ? "Yes" : "No" %> <%= user.disabledReason ? `(${user.disabledReason})` : "" %>

        <% }); %>

      <% if (page > 1) { %>

<a href="/admin/manage/list-users?sort=<%= sort %>&page=<%= page - 1 %>">«

      <% } %> <% for (var i = 1; i <= pageCount; i++) { %>
      <li <% if (i === page) { %>class="active"<% } %>><a href="/admin/manage/list-users?sort=<%= sort %>&page=<%= i %>"><%= i %>

      <% } %> <% if (page < pageCount) { %>

<a href="/admin/manage/list-users?sort=<%= sort %>&page=<%= page + 1 %>">»

      <% } %>

Showing <%= page * pageSize - pageSize + 1 %> to <%= users.length + page * pageSize - pageSize %> of <%= totalCount %> users.

    <%- include("../_partials/pagination") %>
    <% } %>

<%- include("../_partials/admin-footer") %>


code:
<%- include("../_partials/admin-header", { title: "Login" }) %>

Login

    <% if (failed) { %>

Please try again.

    <% } %>
    <form action="/admin/login" method="post">
      <input type="hidden" name="_csrf" value="<%= csrfToken %>" />
      Admin Key
      <input type="password" name="token" />
      <input type="submit" value="Login" />
    </form>


code:
<%- include("../_partials/admin-header", { title: "View User - OAI Reverse Proxy Admin" }) %>

View User

      Key
      Value

      Token
      <%- user.token %>

      Type
      <%- user.type %>

      Prompt Count
      <%- user.promptCount %>

      Token Count
      <%- user.tokenCount %>

      Created At
      <%- user.createdAt %>

      Last Used At
      <%- user.lastUsedAt || "never" %>

      Disabled At
      <%- user.disabledAt %>

      Disabled Reason
      <%- user.disabledReason %>

      IPs

        Show all (<%- user.ip.length %>)

          <% user.ip.forEach((ip) => { %>

<%- ip %>

          <% }) %>

<%- include("../_partials/admin-footer") %>


module: CommonJS

moduleResolution: node

outDir: build


code:
*.bat	text eol=crlf
*.tw	text eol=lf
*.sh	text eol=lf
*.py	text eol=lf
*.txt	text eol=lf
compile	text eol=lf
compile-git	text eol=lf
sanityCheck	text eol=lf

repo_name: evrgentesee/fc

path: .gitattributes


code:
# Byte-compiled / optimized / DLL files
__pycache__/
*.py[cod]
*$py.class
*.swp
*.swo

# C extensions
*.so

# Distribution / packaging
.Python
env/
build/
develop-eggs/
dist/
downloads/
eggs/
.eggs/
lib/
lib64/
parts/
sdist/
var/
*.egg-info/
.installed.cfg
*.egg

# PyInstaller
#  Usually these files are written by a python script from a template
#  before PyInstaller builds the exe, so as to inject date/other infos into it.
*.manifest
*.spec

# Installer logs
pip-log.txt
pip-delete-this-directory.txt

# Unit test / coverage reports
htmlcov/
.tox/
.coverage
.coverage.*
.cache
nosetests.xml
coverage.xml
*,cover
.hypothesis/

# Translations
*.mo
*.pot

# Django stuff:
*.log
local_settings.py

# Flask stuff:
instance/
.webassets-cache

# Scrapy stuff:
.scrapy

# Sphinx documentation
docs/_build/

# PyBuilder
target/

# IPython Notebook
.ipynb_checkpoints

# pyenv
.python-version

# celery beat schedule file
celerybeat-schedule

# dotenv
.env

# virtualenv
venv/
ENV/

# Spyder project settings
.spyderproject

# Rope project settings
.ropeproject

# Start.tw
src/config/start.tw

path: .gitignore


code:
further development:
-specialized slave schools
-fortifications
-more levels for militia edict (further militarize society)
-conquering other arcologies?

Events:
-famous criminal escapes to the arcology, followed by another arcology police force

Bugs:
-sometimes troop counts breaks
-sometimes rebel numbers have fractionary parts

Rules Assistant:
- find a way for the new intense drugs to fit in
- everything mentioned in https://gitgud.io/pregmodfan/fc-pregmod/issues/81

main.tw porting:
- slaveart
- slavesummary
- slavesummaryuncached
- clothingSelectionJS.tw pronouns


code:
# How to vector art

Please read the whole document before starting to work.
Some aspects are not obvious right away.

Note: This does not actually describe how to be an artist.

## TL;DR

    killall inkscape
    artTools/vector_layer_split.py artTools/vector_source.svg tw src/art/vector/layers/
    compile

## 1. Be an artist

Make changes to the vector_source.svg.
Inkscape was thoroughly tested.
Adobe Illustrator might work decently, too.

## 2. Respect the structure

While editing, keep the Layers in mind.

* In Inkscape, Layers are special "Inkscape groups".
* In Illustrator, Layers are groups with user-definded IDs.
* All Layers should have an ID that is globally unique
  (not just within their subtree).

* Please use anonymous groups only for the ease of editing. Remove all "helper" groups before finally saving the file.
* Anonymous groups can be used for continous scaling (of e.g. boobs).

* Every asset that should go into a separate file needs to be in a labelled group
  (even if that is a group with only one shape).
* There are some globally available styles defined as CSS classes (e.g. skin, hair).
  Use them if your asset should be changed upon display.
  Do not set the style directly in the object.

## 3. Normalise the document (before committing)

**THIS IS IMPORTANT**

The various editors out there (Inkscape, Illustrator) may use different variants of formatting the SVG XML.
Use

    python3 normalize_svg.py vector_source.svg

before committing to normalise the format so git will not freak out about the changed indentation.

If you use Inkscape, please use in Edit → Settings → Input/Output → SVG-Output → Path Data → Optimized. This is the default.
In case your Editor uses another path data style which cannot be changed, please contact the other artists and developers via the issue tracker to find a new common ground.

What it does:
* Formats the SVG XML according to Pythons lxml module, regardless of editor.
* Adobe Illustrator uses group IDs as layer labels.
  Inkscape however uses layer labels and a separate, auto-generated group ID.
  normalize_svg.py overwrites the group ID with the Inkscape layer label
  so they are synchronised with Inkscape layer labels.
* Inkscape copies the global style definition into the object *every time*
  the object is edited. If an object references a CSS class AND at the same time
  has a local style, the local style is removed
  so global dynamic styling is possible later on.

Note: Behaviour of Adobe Illustrator is untested.

## 4. Split the layers

For NoX original, deepurk extensions, execute

    python3 vector_layer_split.py vector_source_ndmain.svg tw ../src/art/vector/layers/

For faraen revamped art (based on NoX original)

	python3 vector_revamp_layer_split.py vector_revamp_source.svg tw ../src/art/vector_revamp/layers/

. This application reads all groups in `vector_source.svg`.
Each group is stored in a separate file in the target directory `/src/art/vector/layers/`.
The group ID sets the file name. Therefore, the group ID **must not** contain spaces or any other weird characters.

Also consider:
* A group with ID ending in _ is ignored. Use Layers ending in _ to keep overview.
* A group with ID starting with "g" (Inkscape) or "XMLID" (Illustrator) is also ignored.
* Existing files are overwritten **without enquiry**.
* The target directory is not emptied. If a file is no longer needed, you should remove it manually.
* This procedure removes global definitions. This means, SVG filters are currently not supported.

Available output formats are `svg` and `tw`.
`svg` output exists for debug reasons.
`tw` embeds the SVG data into Twine files, but removes the global style definitions so they can be set during display.

## 5. Edit the code

`/src/art/` contains Twine code which shows the assets in the story.
There are many helpful comments in `/src/art/artWidgets.tw`.
The code also generates the previously removed global style definitions on the fly and per display.

## 6. Compile the story

Use the provided compile script (Windows batch or shell) for compiling.

## 7. Advanced usage

You can define multiple CSS classes to one element, e.g. "skin torso". When procedurally generating CSS, you can then set a global "skin" style, and a different one for "torso".

You can put variable text into the image using single quotes, e.g. "'+_tattooText+'". The single quotes *break* the quoting in Twine so the content of the Twine variable `_tattooText` is shown upon display. You can even align text on paths.

An anonymous group can be used for dynamic transformation. However, finding the correct parameters for the affine transformation matrix can be cumbersome. Use any method you see fit. See `src/art/vector/Boob.tw` for one example of the result.


code:
Apron
AttractiveLingerie
AttractiveLingerieForAPregnantWoman
BallGown
Battledress
BodyOil
Bunny
Chains
ChattelHabit
Cheerleader
ClubslutNetting
ComfortableBodysuit
Conservative
Cutoffs
Cybersuit
FallenNunsHabit
FuckdollSuit
HalterTopDress
HaremGauze
Hijab
Huipil
Kimono
LatexCatsuit
Leotard
LongQipao
MaternityDress
MilitaryUniform
MiniDress
Monokini
NiceBusinessAttire
NiceMaid
NiceNurse
No
PenitentNunsHabit
RedArmyUniform
RestrictiveLatex
ScalemailBikini
SchoolgirlUniform
SchutzstaffelUniform
ShibariRopes
SlaveGown
Slutty
SluttyBusinessAttire
SluttyJewelry
SluttyMaid
SluttyNurse
SluttyQipao
SluttySchutzstaffelUniform
Spats
StretchPants
StringBikini
Succubus
Toga
UncomfortableStraps
Western


code:
#!/bin/bash

# This script reads all possible values of $slave.clothes as mentioned by the documentation.
# This script uses the actual implementation of the JS clothing2artSuffix function as defined in src/art/vector/Helper_Functions.tw
# This script outputs suffixes to be used in the SVG naming scheme
# This script is meant to be executed at the project root directory.
# This script depends on bash, grep, sed, paste and nodejs (so best executed on Linux, I guess)

# grep -Porh '(?<=\.clothes = )"[^"]+"' src/ | sort | uniq # searches sources for all clothes strings actually used in the game, even undocumented ones

(
  echo 'var window = {};'
  grep -v '^:' src/art/vector/Helper_Functions.tw
  echo -n 'Array('
  sed '/^clothes:/,/:/!d' "slave variables documentation - Pregmod.txt" | grep '"' | paste -sd,
  echo ').forEach(v => {console.log(window.clothing2artSuffix(v));});'
) | nodejs | sort


code:
#!/usr/bin/env python3

'''
Application for "normalizing" SVGs

These problems are addressed:

* Inkscape notoriously copies class styles into the object definitions.
https://bugs.launchpad.net/inkscape/+bug/167937

* Inkscape uses labels on layers. Layers are basically named groups.
  Inkscape does not sync the group id with the layer label.

Usage Example:
python3 inkscape_svg_fixup.py vector_source.svg
'''

import lxml.etree as etree
import sys

def fix(tree):
  # know namespaces
  ns = {
    'svg' : 'http://www.w3.org/2000/svg',
    'inkscape' : 'http://www.inkscape.org/namespaces/inkscape'
  }

  # find document global style definition
  # mangle it and interpret as python dictionary
  style_element = tree.find('./svg:style',namespaces=ns)
  style_definitions = style_element.text
  pythonic_style_definitions = '{'+style_definitions.\
  replace('\t.','"').replace('{','":"').replace('}','",').\
  replace('/*','#')+'}'
  styles = eval(pythonic_style_definitions)

  # go through all SVG elements
  for elem in tree.iter():
    if (elem.tag == etree.QName(ns['svg'], 'g')):
      # compare inkscape label with group element ID
      l = elem.get(etree.QName(ns['inkscape'], 'label'))
      if l:
        i = elem.get('id')
        if (i != l):
          print("Overwriting ID %s with Label %s..."%(i, l))
          elem.set('id', l)

    # clean styles (for easier manual merging)
    style_string = elem.get('style')
    if style_string:
      split_styles = style_string.strip('; ').split(';')
      styles_pairs = [s.strip('; ').split(':') for s in split_styles]
      filtered_pairs = [ (k,v) for k,v in styles_pairs if not (
        k.startswith('font-') or
        k.startswith('text-') or
        k.endswith('-spacing') or
        k in ["line-height", " direction", " writing", " baseline-shift", " white-space", " writing-mode"]
      )]
      split_styles = [':'.join(p) for p in filtered_pairs]
      style_string = ';'.join(sorted(split_styles))
      elem.attrib["style"] = style_string

    # remove all style attributes offending class styles
    s = elem.get('style')
    c = elem.get('class')
    if (c and s):
      s = s.lower()
      c = c.split(' ')[0] # regard main style only
      cs = ''
      if c in styles:
        cs = styles[c].strip('; ').lower()
      if (c not in styles):
        print("Object id %s references unknown style class %s."%(i,c))
      else:
        if (cs != s.strip('; ')):
          print("Style %s removed from object id %s differed from class %s style %s."%(s,i,c,cs))
        del elem.attrib["style"]

if __name__ == "__main__":
  input_file = sys.argv[1]
  tree = etree.parse(input_file)
  fix(tree)
  # store SVG into file (input file is overwritten)
  svg = etree.tostring(tree, pretty_print=True)
  with open(input_file, 'wb') as f:
    f.write('<?xml version="1.0" encoding="UTF-8" standalone="no"?>\n'.encode("utf-8"))
    f.write(svg)


code:
#!/usr/bin/env python3

'''
Application for procedural content adaption
*THIS IS VERY EXPERIMENTAL*
Contains a very poor man's implementation of spline mesh warping.

This application parses SVG path data of an outfit sample aligned to a body part.
The outfit is then replicated for other shapes of the same body part.

Example data is geared towards generating a strap outfit for boobs and torso
for all sizes of boobs and all shapes of torsos based on a single outfit for
boobs of size 2 and a hourglass torso respectively.

Limitations:
* handles paths only
* only svg.path Line and CubicBezier are tested
* only the aforementioned examples are tested

Usage:
python3 vector_clothing_replicator.py infile clothing bodypart destinationfile

Usage Example:
python3 vector_clothing_replicator.py vector_source.svg Straps Boob vector_destination.svg
python3 vector_clothing_replicator.py vector_source.svg Straps Torso vector_destination.svg
'''

from svg.path import parse_path
import copy
import lxml.etree as etree
import sys

REFERENCE_PATH_SAMPLES = 200
EMBED_REPLICATIONS = True # wether to embed all replications into the input file or output separate files

input_file = sys.argv[1]
clothing = sys.argv[2]
bodypart = sys.argv[3]
output_file_embed = sys.argv[4]

# TODO: make these configurable
output_file_pattern = '%s_%s_%s.svg' #bodypart, target_id, clothing

if ('Torso' == bodypart):
  xpath_shape = './svg:g[@id="Torso_"]/svg:g[@id="Torso_%s"]/svg:path[@class="skin torso"]/@d' # TODO: formulate more general, independent of style
  xpath_outfit_container = '//svg:g[@id="Torso_Outfit_%s_"]'%(clothing)
  xpath_outfit = '//svg:g[@id="Torso_Outfit_%s_%s"]'%(clothing,'%s')
  target_ids = "Unnatural,Hourglass,Normal".split(",")
  reference_id = "Hourglass"
else:
  raise RuntimeError("Please specify a bodypart for clothing to replicate.")

tree = etree.parse(input_file)
ns = {'svg' : 'http://www.w3.org/2000/svg'}

canvas = copy.deepcopy(tree)
for e in canvas.xpath('./svg:g',namespaces=ns)+canvas.xpath('./svg:path',namespaces=ns):
  # TODO: this should be "remove all objects, preserve document properties"
  e.getparent().remove(e)

def get_points(xpath_shape):
  '''
  This funciton extracts reference paths by the given xpath selector.
  Each path is used to sample a fixed number of points.
  '''
  paths_data = tree.xpath(xpath_shape,namespaces=ns)
  points = []
  path_length = None
  for path_data in paths_data:
    p = parse_path(path_data)
    points += [
      p.point(1.0/float(REFERENCE_PATH_SAMPLES)*i)
      for i in range(REFERENCE_PATH_SAMPLES)
    ]
  if (not points):
    raise RuntimeError(
      'No paths for reference points found by selector "%s".'%(xpath_shape)
    )
  return points

def point_movement(point, reference_points, target_points):
  '''
  For a given point, finds the nearest point in the reference path.
  Gives distance vector from the nearest reference point to the
  respective target reference point.
  '''
  distances = [abs(point-reference_point) for reference_point in reference_points]
  min_ref_dist_idx = min(enumerate(distances), key=lambda x:x[1])[0]
  movement = target_points[min_ref_dist_idx] - reference_points[min_ref_dist_idx]
  return movement

reference_points = get_points(xpath_shape%(reference_id))
container = tree.xpath(xpath_outfit_container,namespaces=ns)
if (len(container) != 1):
  raise RuntimeError('Outfit container selector "%s" does not yield exactly one layer.'%(xpath_outfit_container))
container = container[0]
outfit_source = container.xpath(xpath_outfit%(reference_id),namespaces=ns)
if (len(outfit_source) != 1):
  raise RuntimeError('Outfit source selector "%s" does not yield exactly one outfit layer in container selected by "%s".'%(xpath_outfit%(reference_id), xpath_outfit_container))
outfit_source = outfit_source[0]

for target_id in target_ids:
  print(
    'Generating variant "%s" of clothing "%s" for bodypart "%s"...'%
    (target_id, clothing, bodypart)
  )
  outfit = copy.deepcopy(outfit_source)
  paths = outfit.xpath('./svg:path',namespaces=ns)
  if target_id == reference_id:
    print("This is the source variant. Skipping...")
  else:
    layerid = outfit.get('id').replace('_%s'%(reference_id),'_%s'%(target_id))
    outfit.set('id', layerid)
    outfit.set(etree.QName('http://www.inkscape.org/namespaces/inkscape', 'label'), layerid) # for the Inkscape-users
    target_points = get_points(xpath_shape%(target_id))
    if (len(reference_points) != len(target_points)):
      raise RuntimeError(
        ('Different amounts of sampled points in reference "%s" and target "%s" paths. '+
        'Selector "%s" probably matches different number of paths in the two layers.')%
        (reference_id, target_id, xpath_shape)
      )
    for path in paths:
      path_data = path.get("d")
      p = parse_path(path_data)
      for segment in p:
        original_distance = abs(segment.end-segment.start)
        start_movement = point_movement(segment.start, reference_points, target_points)
        segment.start += start_movement
        end_movement = point_movement(segment.end, reference_points, target_points)
        segment.end += end_movement
        distance = abs(segment.end-segment.start)
        try:
          # enhance position of CubicBezier control points
          # amplification is relative to the distance gained by movement
          segment.control1 += start_movement
          segment.control1 += (segment.control1-segment.start)*(distance/original_distance-1.0)
          segment.control2 += end_movement
          segment.control2 += (segment.control2-segment.end)*(distance/original_distance-1.0)
        except AttributeError as ae:
          # segment is not a CubicBezier
          pass
      path.set("d", p.d())
    if EMBED_REPLICATIONS:
      container.append(outfit)
  if not EMBED_REPLICATIONS:
    container = copy.deepcopy(canvas).xpath('.',namespaces=ns)[0]
    container.append(outfit)

  if not EMBED_REPLICATIONS:
    svg = etree.tostring(container, pretty_print=True)
    with open((output_file_pattern%(bodypart, target_id, clothing)).lower(), 'wb') as f:
      f.write('<?xml version="1.0" encoding="UTF-8" standalone="no"?>\n'.encode("utf-8"))
      f.write(svg)

if EMBED_REPLICATIONS:
  svg = etree.tostring(tree, pretty_print=True)
  with open(output_file_embed, 'wb') as f:
    f.write('<?xml version="1.0" encoding="UTF-8" standalone="no"?>\n'.encode("utf-8"))
    f.write(svg)


code:
#!/usr/bin/env python3

'''
Application for splitting groups from one SVG file into separate files

Usage:
python3 vector_layer_split.py infile format outdir

Usage Example:
python3 vector_layer_split.py vector_source.svg tw ../src/art/vector/layers/
'''

import lxml.etree as etree
import sys
import os
import copy
import re
import normalize_svg

input_file = sys.argv[1]
output_format = sys.argv[2]
output_directory = sys.argv[3]
if not os.path.exists(output_directory):
    os.makedirs(output_directory)

ns = {
  'svg' : 'http://www.w3.org/2000/svg',
  'inkscape' : 'http://www.inkscape.org/namespaces/inkscape',
  'sodipodi':"http://sodipodi.sourceforge.net/DTD/sodipodi-0.dtd",
}

tree = etree.parse(input_file)
normalize_svg.fix(tree)

# prepare output template
template = copy.deepcopy(tree)
root = template.getroot()

# remove all svg root attributes except document size
for a in root.attrib:
  if (a != "viewBox"):
    del root.attrib[a]

# add placeholder for CSS class (needed for workaround for non HTML 5.1 compliant browser)
if output_format == 'tw':
  root.attrib["class"] = "'+_art_display_class+'"

# remove all content, including metadata
# for twine output, style definitions are removed, too
defs = None
for e in root:
  if (e.tag == etree.QName(ns['svg'], 'defs')):
    defs = e
  if (e.tag == etree.QName(ns['svg'], 'g') or
      e.tag == etree.QName(ns['svg'], 'metadata') or
      e.tag == etree.QName(ns['svg'], 'defs') or
      e.tag == etree.QName(ns['sodipodi'], 'namedview') or
      (output_format == 'tw' and e.tag == etree.QName(ns['svg'], 'style'))
      ):
    root.remove(e)
# template preparation finished

# prepare regex for later use
regex_xmlns = re.compile(' xmlns[^ ]+',)
regex_space = re.compile('[>][ ]+[<]',)

# find all groups
layers = tree.xpath('//svg:g',namespaces=ns)
for layer in layers:
  i = layer.get('id')
  if ( # disregard non-content groups
    i.endswith("_") or # manually suppressed with underscore
    i.startswith("XMLID") or # Illustrator generated group
    i.startswith("g") # Inkscape generated group
  ):
    continue
  # create new canvas
  output = copy.deepcopy(template)
  # copy all shapes into template
  canvas = output.getroot()
  for e in layer:
    canvas.append(e)
  # represent template as SVG (binary string)
  svg = etree.tostring(output, pretty_print=False)
  # poor man's conditional defs insertion
  # TODO: extract only referenced defs (filters, gradients, ...)
  # TODO: detect necessity by traversing the elements. do not stupidly search in the string representation
  if ("filter:" in svg.decode('utf-8')):
    # it seems there is a filter referenced in the generated SVG, re-insert defs from main document
    canvas.insert(0,defs)
    # re-generate output
    svg = etree.tostring(output, pretty_print=False)

  if (output_format == 'tw'):
    # remove unnecessary attributes
    # TODO: never generate unnecessary attributes in the first place
    svg = svg.decode('utf-8')
    svg = regex_xmlns.sub('',svg)
    svg = svg.replace(' inkscape:connector-curvature="0"','') # this just saves space
    svg = svg.replace('\n','').replace('\r','') # print cannot be multi-line
    svg = regex_space.sub('><',svg) # remove indentaion
    svg = svg.replace('svg:','') # svg namespace was removed
    svg = svg.replace('<g ','<g transform="\'+_art_transform+\'"') # internal groups are used for scaling
    svg = svg.encode('utf-8')

  # save SVG string to file
  i = layer.get('id')
  output_path = os.path.join(output_directory,i+'.'+output_format)
  with open(output_path, 'wb') as f:
    if (output_format == 'svg'):
      # Header for normal SVG (XML)
      f.write('<?xml version="1.0" encoding="UTF-8" standalone="no"?>\n'.encode("utf-8"))
      f.write(svg)
    elif (output_format == 'tw'):
      # Header for SVG in Twine file (SugarCube print statement)
      f.write((':: Art_Vector_%s [nobr]\n\n'%(i)).encode("utf-8"))
      f.write("<<print '<html>".encode("utf-8"))
      f.write(svg)
      f.write("</html>' >>".encode("utf-8"))