mini-data-warehouse/flake.nix at master · timothynn/mini-data-warehouse · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
{
  description = "Mini Data Warehouse - PostgreSQL data warehouse with synthetic e-commerce data";

  inputs = {
    nixpkgs.url = "github:NixOS/nixpkgs/nixos-unstable";
    flake-utils.url = "github:numtide/flake-utils";
  };

  outputs = { self, nixpkgs, flake-utils }:
    flake-utils.lib.eachDefaultSystem (system:
      let
        pkgs = nixpkgs.legacyPackages.${system};

        pythonEnv = pkgs.python3.withPackages (ps: with ps; [
          pandas
          faker
          psycopg2
          matplotlib
          numpy
        ]);
      in
      {
        devShells.default = pkgs.mkShell {
          buildInputs = with pkgs; [
            # Python and dependencies
            pythonEnv

            # Docker and container tools
            docker
            docker-compose

            # PostgreSQL client tools
            postgresql

            # Development tools
            git
          ];

          shellHook = ''
            echo "🏗️  Mini Data Warehouse Development Environment"
            echo ""
            echo "Available tools:"
            echo "  • Python ${pythonEnv.python.version} with pandas, faker, psycopg2, matplotlib"
            echo "  • Docker and Docker Compose"
            echo "  • PostgreSQL client tools (psql)"
            echo ""
            echo "Quick start:"
            echo "  1. Generate data:         python data.py"
            echo "  2. Start services:        docker-compose up -d"
            echo "  3. Setup star schema:     python -c \"import psycopg2; conn=psycopg2.connect('host=localhost port=5432 dbname=warehouse user=admin password=secret'); cur=conn.cursor(); cur.execute(open('star_schema.sql').read()); conn.commit()\""
            echo "  4. Run transformations:   python transform_pipeline.py"
            echo "  5. Check data quality:    python data_quality.py"
            echo "  6. Generate dashboard:    python dashboard.py"
            echo "  7. Access PgAdmin:        http://localhost:8080"
            echo ""
            echo "Management commands:"
            echo "  • Incremental load:       python incremental_load.py"
            echo "  • Retention policies:     python retention_policy.py --execute"
            echo "  • Quality checks:         python data_quality.py"
            echo ""
            echo "💡 Run 'docker-compose down' to stop services when done"
          '';
        };

        # Package for the data generation script
        packages.default = pkgs.stdenv.mkDerivation {
          pname = "mini-data-warehouse";
          version = "1.0.0";

          src = ./.;

          buildInputs = [ pythonEnv ];

          installPhase = ''
            mkdir -p $out/bin $out/share/mini-data-warehouse

            # Copy all project files
            cp -r * $out/share/mini-data-warehouse/

            # Create wrapper script for data generation
            cat > $out/bin/generate-data << 'EOF'
            #!/bin/sh
            cd $out/share/mini-data-warehouse
            mkdir -p data
            ${pythonEnv}/bin/python data.py
            EOF
            chmod +x $out/bin/generate-data
          '';

          meta = with pkgs.lib; {
            description = "Mini data warehouse with synthetic e-commerce data";
            homepage = "https://github.com/user/mini-data-warehouse";
            license = licenses.mit;
            maintainers = [ ];
          };
        };

        # Docker image (optional, for containerized deployment)
        packages.docker = pkgs.dockerTools.buildImage {
          name = "mini-data-warehouse";
          tag = "latest";

          contents = [ self.packages.${system}.default ];

          config = {
            Cmd = [ "${self.packages.${system}.default}/bin/generate-data" ];
            WorkingDir = "/app";
          };
        };

        # App for running the data generator
        apps.default = {
          type = "app";
          program = "${self.packages.${system}.default}/bin/generate-data";
        };

        apps.generate-data = {
          type = "app";
          program = "${self.packages.${system}.default}/bin/generate-data";
        };
      });
}