Move the CLI hack to a dedicated notebook

from datetime import datetime

from os import environ, popen

import queue

import re

import subprocess

import threading

import time

import ipywidgets as widgets

    display(widget_layout)

# Function to read output continuously from stdout and stderr

def read_output(stream, output_queue):

    for line in iter(stream.readline, ''):

        output_queue.put(line)

    stream.close()

class BeelineCLI:

    def __init__(self):

        # Start the beeline CLI as a subprocess

        self.beeline_process = subprocess.Popen(

            ['beeline'],

            stdin=subprocess.PIPE,

            stdout=subprocess.PIPE,

            stderr=subprocess.PIPE,

            text=True,

            bufsize=1

        )

        # Queues to hold stdout and stderr lines

        self.stdout_queue = queue.Queue()

        self.stderr_queue = queue.Queue()

        # Threads to read from stdout and stderr

        self.stdout_thread = threading.Thread(

            target=read_output,

            args=(

                self.beeline_process.stdout,

                self.stdout_queue

            ), daemon=True

        )

        self.stderr_thread = threading.Thread(

            target=read_output,

            args=(

                self.beeline_process.stderr,

                self.stderr_queue

            ), daemon=True

        )

        self.stdout_thread.start()

        self.stderr_thread.start()

        self.print_outputs()

    def run(self, command, wait_s=1):

        """Send a query to the beeline process and print the output."""

        # Write the query to beeline's stdin

        self.beeline_process.stdin.write(command + '\n')

        self.beeline_process.stdin.flush()

        self.print_outputs(wait_s=wait_s)

    def print_outputs(self, wait_s=1):

        last_output_handled = datetime.now()

        while True:

            output_handled = False

            try:

                while not self.stderr_queue.empty():

                    print(self.stderr_queue.get_nowait(), end='')

                    output_handled = True

                    last_output_handled = datetime.now()

                while not self.stdout_queue.empty():

                    print(self.stdout_queue.get_nowait(), end='')

                    output_handled = True

                    last_output_handled = datetime.now()

            except queue.Empty:

                continue

            # Break the loop if no new output is being handled

            last_handling_s = (datetime.now() - last_output_handled).seconds

            if (not output_handled) and (last_handling_s > wait_s):

                break

    def terminate(self):

        # Stop the beeline process after use

        self.run('!quit')

        self.beeline_process.terminate()

        self.stdout_thread.join()

        self.stderr_thread.join()

set_hadoop_env()

kinit()

 No newline at end of file

%% Cell type:code id:69b71081-968d-4e98-995f-057d28b6fa6a tags:

``` python

%load_ext autoreload

```

%% Cell type:code id:9b51aabc-bbe9-4a8f-af43-f1d2de1cdd45 tags:

``` python

%autoreload 1

```

%% Cell type:code id:8942713d-49f7-408a-a2ff-242914455da0 tags:

``` python

%aimport ui_utils

```

%% Cell type:markdown id:69c2975e-a7af-4fa1-a423-f4604e275d2f tags:

# Running a CLI application from a notebook

<p style="font-size:36px; color:red; font-weight:bold">Hack alert</p>

This has been done for demostrative purposes but it is not really convenient and safe. Only use this as the last option!!

This notebook and the realted library contains some code to be able to interact with CLI utilities from a notebook

%% Cell type:code id:9727ec1a-96de-4172-b93d-9f05aa858d8d tags:

``` python

cli = ui_utils.CLI(['/bin/sqlite3'], 1)

```

%% Cell type:code id:636487b8-4825-4ccb-a2d3-37bce3a31970 tags:

``` python

cli.run('.help')

```

%% Output

    .archive ...             Manage SQL archives

    .auth ON|OFF             Show authorizer callbacks

    .backup ?DB? FILE        Backup DB (default "main") to FILE

    .bail on|off             Stop after hitting an error.  Default OFF

    .binary on|off           Turn binary output on or off.  Default OFF

    .cd DIRECTORY            Change the working directory to DIRECTORY

    .changes on|off          Show number of rows changed by SQL

    .check GLOB              Fail if output since .testcase does not match

    .clone NEWDB             Clone data into NEWDB from the existing database

    .databases               List names and files of attached databases

    .dbconfig ?op? ?val?     List or change sqlite3_db_config() options

    .dbinfo ?DB?             Show status information about the database

    .dump ?TABLE?            Render database content as SQL

    .echo on|off             Turn command echo on or off

    .eqp on|off|full|...     Enable or disable automatic EXPLAIN QUERY PLAN

    .excel                   Display the output of next command in spreadsheet

    .exit ?CODE?             Exit this program with return-code CODE

    .expert                  EXPERIMENTAL. Suggest indexes for queries

    .explain ?on|off|auto?   Change the EXPLAIN formatting mode.  Default: auto

    .filectrl CMD ...        Run various sqlite3_file_control() operations

    .fullschema ?--indent?   Show schema and the content of sqlite_stat tables

    .headers on|off          Turn display of headers on or off

    .help ?-all? ?PATTERN?   Show help text for PATTERN

    .import FILE TABLE       Import data from FILE into TABLE

    .imposter INDEX TABLE    Create imposter table TABLE on index INDEX

    .indexes ?TABLE?         Show names of indexes

    .limit ?LIMIT? ?VAL?     Display or change the value of an SQLITE_LIMIT

    .lint OPTIONS            Report potential schema issues.

    .load FILE ?ENTRY?       Load an extension library

    .log FILE|off            Turn logging on or off.  FILE can be stderr/stdout

    .mode MODE ?TABLE?       Set output mode

    .nullvalue STRING        Use STRING in place of NULL values

    .once ?OPTIONS? ?FILE?   Output for the next SQL command only to FILE

    .open ?OPTIONS? ?FILE?   Close existing database and reopen FILE

    .output ?FILE?           Send output to FILE or stdout if FILE is omitted

    .parameter CMD ...       Manage SQL parameter bindings

    .print STRING...         Print literal STRING

    .progress N              Invoke progress handler after every N opcodes

    .prompt MAIN CONTINUE    Replace the standard prompts

    .quit                    Exit this program

    .read FILE               Read input from FILE

    .recover                 Recover as much data as possible from corrupt db.

    .restore ?DB? FILE       Restore content of DB (default "main") from FILE

    .save FILE               Write in-memory database into FILE

    .scanstats on|off        Turn sqlite3_stmt_scanstatus() metrics on or off

    .schema ?PATTERN?        Show the CREATE statements matching PATTERN

    .selftest ?OPTIONS?      Run tests defined in the SELFTEST table

    .separator COL ?ROW?     Change the column and row separators

    .sha3sum ...             Compute a SHA3 hash of database content

    .shell CMD ARGS...       Run CMD ARGS... in a system shell

    .show                    Show the current values for various settings

    .stats ?on|off?          Show stats or turn stats on or off

    .system CMD ARGS...      Run CMD ARGS... in a system shell

    .tables ?TABLE?          List names of tables matching LIKE pattern TABLE

    .testcase NAME           Begin redirecting output to 'testcase-out.txt'

    .testctrl CMD ...        Run various sqlite3_test_control() operations

    .timeout MS              Try opening locked tables for MS milliseconds

    .timer on|off            Turn SQL timer on or off

    .trace ?OPTIONS?         Output each SQL statement as it is run

    .vfsinfo ?AUX?           Information about the top-level VFS

    .vfslist                 List all available VFSes

    .vfsname ?AUX?           Print the name of the VFS stack

    .width NUM1 NUM2 ...     Set minimum column widths for columnar output

%% Cell type:code id:597f8500-6dd7-42af-ac29-c2b5ba57a16e tags:

``` python

```

from datetime import datetime

import queue

import subprocess

import threading

# Function to read output continuously from stdout and stderr

def read_output(stream, output_queue):

    for line in iter(stream.readline, ''):

        output_queue.put(line)

    stream.close()

class CLI:

    def __init__(self, cmd, init_wait_s=1):

        # Start the CLI as a subprocess

        self.cli_process = subprocess.Popen(

            cmd,

            stdin=subprocess.PIPE,

            stdout=subprocess.PIPE,

            stderr=subprocess.PIPE,

            text=True,

            bufsize=1

        )

        # Queues to hold stdout and stderr lines

        self.stdout_queue = queue.Queue()

        self.stderr_queue = queue.Queue()

        # Threads to read from stdout and stderr

        self.stdout_thread = threading.Thread(

            target=read_output,

            args=(

                self.cli_process.stdout,

                self.stdout_queue

            ), daemon=True

        )

        self.stderr_thread = threading.Thread(

            target=read_output,

            args=(

                self.cli_process.stderr,

                self.stderr_queue

            ), daemon=True

        )

        self.stdout_thread.start()

        self.stderr_thread.start()

        self.print_outputs(wait_s=init_wait_s)

    def run(self, command, wait_s=1):

        """Send a command to the process and print the output."""

        self.cli_process.stdin.write(command + '\n')

        self.cli_process.stdin.flush()

        self.print_outputs(wait_s=wait_s)

    def print_outputs(self, wait_s=1):

        last_output_handled = datetime.now()

        while True:

            output_handled = False

            try:

                while not self.stderr_queue.empty():

                    print(self.stderr_queue.get_nowait(), end='')

                    output_handled = True

                    last_output_handled = datetime.now()

                while not self.stdout_queue.empty():

                    print(self.stdout_queue.get_nowait(), end='')

                    output_handled = True

                    last_output_handled = datetime.now()

            except queue.Empty:

                continue

            # Break the loop if no new output is being handled

            last_handling_s = (datetime.now() - last_output_handled).seconds

            if (not output_handled) and (last_handling_s > wait_s):

                break

    def terminate(self, command=None):

        if not command is None:

            self.run(command)

        self.cli_process.terminate()

        self.stdout_thread.join()

        self.stderr_thread.join()

 No newline at end of file