feat(agy): add keyless image-analyze + search modes
image-analyze: independent second-model vision over OAuth (pins the gemini-3.1-pro-preview vision model; the default flash-lite router hallucinates image content) — reads an image via read_file and describes it. search: Google-grounded live web results with citation URLs (google_web_search). Both verified working on the keyless Google OAuth. Image GENERATION (nano-banana) still needs an AI Studio key + extension and stays Grok's lane. Includes a scoped best-effort output sanitizer for image-analyze (preview model occasionally leaks reasoning tokens); text/verify/review/search unchanged. migrate-identity.sh now upgrades the gemini capabilities array. Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
@@ -32,6 +32,8 @@
|
||||
# ask-gemini.sh review <file> [instructions] # gemini reads + reviews one file
|
||||
# ask-gemini.sh review-files [-i "instr"] <f1> [f2 ...] # review a SET of files together
|
||||
# ask-gemini.sh review-diff [-C <repo-dir>] [-i "instr"] <gitref> [-- <pathspec>]
|
||||
# ask-gemini.sh image-analyze <image-path> ["question"] # vision: read_file image + describe (PRO model)
|
||||
# ask-gemini.sh search "<query>" # Google-grounded live web search + sources
|
||||
# ask-gemini.sh raw <gemini args...> # escape hatch
|
||||
#
|
||||
# Exit: 0 ok, 1 no result, 2 usage, 3 not installed here, 127 gemini/python not found.
|
||||
@@ -97,7 +99,7 @@ fi
|
||||
STRONG_MODEL="${GEMINI_MODEL:-gemini-3.1-pro-preview}"
|
||||
|
||||
MODE="${1:-}"; shift 2>/dev/null || true
|
||||
[ -z "$MODE" ] && { echo "usage: $SELF {text|verify|review|review-files|review-diff|raw} ..." >&2; exit 2; }
|
||||
[ -z "$MODE" ] && { echo "usage: $SELF {text|verify|review|review-files|review-diff|image-analyze|search|raw} ..." >&2; exit 2; }
|
||||
|
||||
TMP="$(mktemp -d)"; trap 'rm -rf "$TMP"' EXIT
|
||||
PF="$TMP/prompt.txt"; OUT="$TMP/out.txt"; ERR="$TMP/err.txt"
|
||||
@@ -123,15 +125,62 @@ run_gemini() {
|
||||
|
||||
# extract .response from the JSON object starting at the first '{' in $OUT.
|
||||
# Parsed via stdin so Windows python never resolves a git-bash (/c/...) path.
|
||||
gresponse() { "$PY" -c "import json,sys
|
||||
#
|
||||
# Some pinned-pro tool-using turns (notably image-analyze) leak the model's
|
||||
# internal reasoning stream into .response: a stray token + a 'thought' marker
|
||||
# followed by 'CRITICAL INSTRUCTION N:' lines, then the real answer. We strip
|
||||
# that preamble ONLY when the signature is clearly present, so clean responses
|
||||
# (text/verify/review/search) pass through byte-for-byte unchanged.
|
||||
gresponse() { "$PY" -c "import json,sys,re,os
|
||||
raw=sys.stdin.read()
|
||||
i=raw.find('{')
|
||||
if i < 0:
|
||||
print(''); sys.exit(0)
|
||||
try:
|
||||
print(json.loads(raw[i:]).get('response','') or '')
|
||||
r=json.loads(raw[i:]).get('response','') or ''
|
||||
except Exception:
|
||||
print('')" < "$OUT"; }
|
||||
print(''); sys.exit(0)
|
||||
head=r[:40].lower()
|
||||
leak=('thought' in head) or ('critical instruction' in r.lower()[:600])
|
||||
if leak:
|
||||
lines=r.split('\n')
|
||||
keep=[]; dropping=True
|
||||
for ln in lines:
|
||||
s=ln.strip()
|
||||
low=s.lower()
|
||||
if dropping and (
|
||||
low.endswith('thought') or low.startswith('critical instruction')
|
||||
or low.startswith('thought:') or low=='' ):
|
||||
continue
|
||||
dropping=False
|
||||
keep.append(ln)
|
||||
cleaned='\n'.join(keep).strip()
|
||||
r=cleaned if cleaned else r.strip()
|
||||
# AGY_CLEAN: aggressive prefix scrub for tool-using turns (image-analyze), which
|
||||
# can fuse a stray stream/tool token onto the front of the answer (e.g. '.',
|
||||
# '.94>', 'uem_image_0_0_png}'). Off by default so text/verify/review/search are
|
||||
# byte-exact. We only remove a junk run that ends in a stream delimiter (} > :)
|
||||
# or a lone leading punctuation char, immediately before the first real sentence.
|
||||
if os.environ.get('AGY_CLEAN') == '1' and r:
|
||||
# The pro-preview tool loop sometimes prepends a numbered/markdown reasoning
|
||||
# block before the actual answer. If a clear answer pivot follows such a
|
||||
# preamble, keep from the pivot onward (the user-facing answer).
|
||||
if re.search(r'(?im)^\s*\d+[.)]\s', r) or 'thought' in r[:60].lower():
|
||||
pivs=list(re.finditer(r'(?i)(Based on the image\b|\*\*Answer:?\*\*|The image (?:contains|shows|displays)\b)', r))
|
||||
if pivs:
|
||||
r=r[pivs[-1].start():]
|
||||
m=re.match(r'^[^\n]{0,40}?(?:\.png\)|\.jpe?g\)|[}>:)])\s*([\"A-Z].*)$', r, re.S)
|
||||
if m and m.group(1):
|
||||
r=m.group(1)
|
||||
else:
|
||||
# a short leading junk run (ASCII punctuation/digits or non-Latin stream
|
||||
# tokens) before a capitalized/quoted sentence start. Bounded length so we
|
||||
# never eat a real lowercase sentence or real prose.
|
||||
m=re.match(r'^(?:[^A-Za-z\"]|[^\x00-\x7f]){1,8}([A-Z\"].*)$', r, re.S)
|
||||
if m and m.group(1):
|
||||
r=m.group(1)
|
||||
r=r.strip()
|
||||
print(r)" < "$OUT"; }
|
||||
|
||||
# detect an auth failure in stderr (so we can give a precise remediation hint)
|
||||
auth_failed() { grep -qiE 'oauth|unauthor|authenticat|login|credential|invalid_grant|401' "$ERR" 2>/dev/null; }
|
||||
@@ -263,10 +312,49 @@ case "$MODE" in
|
||||
emit_or_fail
|
||||
;;
|
||||
|
||||
image-analyze|image|vision)
|
||||
# Independent second-model VISION. The default flash-lite router hallucinates
|
||||
# image content, so we PIN the pro vision model (STRONG_MODEL) and run with
|
||||
# yolo approval so read_file can execute. The image is copied into an included
|
||||
# temp dir (like the review modes) and handed to Gemini by absolute winpath.
|
||||
[ -z "${1:-}" ] && { echo "usage: $SELF image-analyze <image-path> [\"question\"]" >&2; exit 2; }
|
||||
target="$1"
|
||||
question="${2:-Describe exactly what is in this image.}"
|
||||
if [ -f "$target" ]; then resolved="$target"
|
||||
elif [ -f "$REPO_ROOT/$target" ]; then resolved="$REPO_ROOT/$target"
|
||||
else echo "[$SELF] image not found: $target" >&2; exit 2; fi
|
||||
prep_includes
|
||||
base="$(basename "$resolved")"
|
||||
cp -f "$resolved" "$INCLUDE_DIR/$base"
|
||||
img_win="$(winpath "$INCLUDE_DIR/$base")"
|
||||
inc_win="$(winpath "$INCLUDE_DIR")"
|
||||
# Image path goes in via %s (never as a printf format string).
|
||||
printf 'Use your read_file tool to read the image at this absolute path, then describe exactly what you see. Report only what is actually present in the image; do not guess or invent content. Then stop. Do not modify anything.\nImage path: %s\n\nQuestion: %s' "$img_win" "$question" > "$PF"
|
||||
run_gemini 240 -m "$STRONG_MODEL" --approval-mode yolo --include-directories "$inc_win"
|
||||
AGY_CLEAN=1 emit_or_fail
|
||||
;;
|
||||
|
||||
search|websearch)
|
||||
# Google-grounded LIVE web search (mirrors grok xsearch). Gemini's
|
||||
# google_web_search tool works on OAuth; run with yolo so the tool can fire.
|
||||
# Query goes via the prompt file so long queries don't hit shell-quote limits.
|
||||
SRC=""
|
||||
if [ "${1:-}" = "--prompt-file" ]; then
|
||||
[ -f "${2:-}" ] || { echo "[$SELF] prompt file not found: ${2:-}" >&2; exit 2; }
|
||||
SRC="$(cat "$2")"
|
||||
else
|
||||
SRC="${1:-}"
|
||||
fi
|
||||
[ -z "$SRC" ] && { echo "usage: $SELF search \"<query>\" | $SELF search --prompt-file <path>" >&2; exit 2; }
|
||||
printf 'Use your google_web_search tool to find current, live information answering the following, then stop. Answer concisely and ALWAYS include the source URLs you used (a Sources list of full URLs). Do not fabricate URLs.\n\nQuery: %s' "$SRC" > "$PF"
|
||||
run_gemini 180 -m "$STRONG_MODEL" --approval-mode yolo
|
||||
emit_or_fail
|
||||
;;
|
||||
|
||||
raw)
|
||||
"$GEMINI" "$@"
|
||||
;;
|
||||
|
||||
*)
|
||||
echo "[$SELF] unknown mode '$MODE' (use text|verify|review|review-files|review-diff|raw)" >&2; exit 2 ;;
|
||||
echo "[$SELF] unknown mode '$MODE' (use text|verify|review|review-files|review-diff|image-analyze|search|raw)" >&2; exit 2 ;;
|
||||
esac
|
||||
|
||||
Reference in New Issue
Block a user