[ { "startMs": 1309, "text": "[music]" }, { "startMs": 12559, "text": "So, I'm going to be talking today about" }, { "startMs": 14709, "text": "So, I'm going to be talking today about" }, { "startMs": 14719, "text": "So, I'm going to be talking today about software factories and uh specifically" }, { "startMs": 18710, "text": "software factories and uh specifically" }, { "startMs": 18720, "text": "software factories and uh specifically like an infrastructure approach to" }, { "startMs": 20150, "text": "like an infrastructure approach to" }, { "startMs": 20160, "text": "like an infrastructure approach to software factories. Um, a little bit" }, { "startMs": 23189, "text": "software factories. Um, a little bit" }, { "startMs": 23199, "text": "software factories. Um, a little bit about me though before we start. So, I'm" }, { "startMs": 25029, "text": "about me though before we start. So, I'm" }, { "startMs": 25039, "text": "about me though before we start. So, I'm Zach. Uh, I've been an engineer for" }, { "startMs": 28550, "text": "Zach. Uh, I've been an engineer for" }, { "startMs": 28560, "text": "Zach. Uh, I've been an engineer for really for a long time at this point. I" }, { "startMs": 30150, "text": "really for a long time at this point. I" }, { "startMs": 30160, "text": "really for a long time at this point. I was a principal engineer at Google. I" }, { "startMs": 31669, "text": "was a principal engineer at Google. I" }, { "startMs": 31679, "text": "was a principal engineer at Google. I used to lead engineering on the Google" }, { "startMs": 32950, "text": "used to lead engineering on the Google" }, { "startMs": 32960, "text": "used to lead engineering on the Google Doc Suite. Helped scale that up to" }, { "startMs": 35430, "text": "Doc Suite. Helped scale that up to" }, { "startMs": 35440, "text": "Doc Suite. Helped scale that up to hundreds of millions and I think maybe" }, { "startMs": 37110, "text": "hundreds of millions and I think maybe" }, { "startMs": 37120, "text": "hundreds of millions and I think maybe now over a billion people who use it." }, { "startMs": 39350, "text": "now over a billion people who use it." }, { "startMs": 39360, "text": "now over a billion people who use it. I'm currently the founder and CEO at" }, { "startMs": 41590, "text": "I'm currently the founder and CEO at" }, { "startMs": 41600, "text": "I'm currently the founder and CEO at Warp. Uh, Warp is a company in the" }, { "startMs": 44150, "text": "Warp. Uh, Warp is a company in the" }, { "startMs": 44160, "text": "Warp. Uh, Warp is a company in the agentic development space. Our current" }, { "startMs": 46150, "text": "agentic development space. Our current" }, { "startMs": 46160, "text": "agentic development space. Our current focus is on helping companies build" }, { "startMs": 49190, "text": "focus is on helping companies build" }, { "startMs": 49200, "text": "focus is on helping companies build their own internal software factories." }, { "startMs": 51029, "text": "their own internal software factories." }, { "startMs": 51039, "text": "their own internal software factories. So, we have a platform called Warp" }, { "startMs": 53270, "text": "So, we have a platform called Warp" }, { "startMs": 53280, "text": "So, we have a platform called Warp Factories which we do that with. We're" }, { "startMs": 55350, "text": "Factories which we do that with. We're" }, { "startMs": 55360, "text": "Factories which we do that with. We're also uh pretty well known for Warp" }, { "startMs": 57189, "text": "also uh pretty well known for Warp" }, { "startMs": 57199, "text": "also uh pretty well known for Warp Terminal, which is an Agentic terminal." }, { "startMs": 59750, "text": "Terminal, which is an Agentic terminal." }, { "startMs": 59760, "text": "Terminal, which is an Agentic terminal. It's open source, around 65,000 GitHub" }, { "startMs": 61990, "text": "It's open source, around 65,000 GitHub" }, { "startMs": 62000, "text": "It's open source, around 65,000 GitHub stars, almost a million active" }, { "startMs": 63349, "text": "stars, almost a million active" }, { "startMs": 63359, "text": "stars, almost a million active developers. Um, our focus as a company" }, { "startMs": 66789, "text": "developers. Um, our focus as a company" }, { "startMs": 66799, "text": "developers. Um, our focus as a company has always been helping developers ship" }, { "startMs": 69590, "text": "has always been helping developers ship" }, { "startMs": 69600, "text": "has always been helping developers ship better software more quickly." }, { "startMs": 72310, "text": "better software more quickly." }, { "startMs": 72320, "text": "better software more quickly. Just to set some context on where we're" }, { "startMs": 75750, "text": "Just to set some context on where we're" }, { "startMs": 75760, "text": "Just to set some context on where we're at uh in terms of agentic development." }, { "startMs": 78310, "text": "at uh in terms of agentic development." }, { "startMs": 78320, "text": "at uh in terms of agentic development. So most companies that I speak with uh" }, { "startMs": 82149, "text": "So most companies that I speak with uh" }, { "startMs": 82159, "text": "So most companies that I speak with uh are at this like phase of what I think" }, { "startMs": 84149, "text": "are at this like phase of what I think" }, { "startMs": 84159, "text": "are at this like phase of what I think of as interactive prompting. And so" }, { "startMs": 86310, "text": "of as interactive prompting. And so" }, { "startMs": 86320, "text": "of as interactive prompting. And so they've deployed tools like cloud code" }, { "startMs": 88310, "text": "they've deployed tools like cloud code" }, { "startMs": 88320, "text": "they've deployed tools like cloud code or codeex to their engineers. Uh" }, { "startMs": 90630, "text": "or codeex to their engineers. Uh" }, { "startMs": 90640, "text": "or codeex to their engineers. Uh engineers are working on their laptops." }, { "startMs": 92390, "text": "engineers are working on their laptops." }, { "startMs": 92400, "text": "engineers are working on their laptops. They are they might have a bunch of" }, { "startMs": 93910, "text": "They are they might have a bunch of" }, { "startMs": 93920, "text": "They are they might have a bunch of these going but generally they're like" }, { "startMs": 95910, "text": "these going but generally they're like" }, { "startMs": 95920, "text": "these going but generally they're like interactively prompting agents to build" }, { "startMs": 97910, "text": "interactively prompting agents to build" }, { "startMs": 97920, "text": "interactively prompting agents to build features and fix bugs. And where uh most" }, { "startMs": 102310, "text": "features and fix bugs. And where uh most" }, { "startMs": 102320, "text": "features and fix bugs. And where uh most companies I speak with want to get to is" }, { "startMs": 104870, "text": "companies I speak with want to get to is" }, { "startMs": 104880, "text": "companies I speak with want to get to is something that is more like automated" }, { "startMs": 106389, "text": "something that is more like automated" }, { "startMs": 106399, "text": "something that is more like automated development. So" }, { "startMs": 108630, "text": "development. So" }, { "startMs": 108640, "text": "development. So the reason for that is really there's" }, { "startMs": 112069, "text": "the reason for that is really there's" }, { "startMs": 112079, "text": "the reason for that is really there's this um you know they want to get to a" }, { "startMs": 114550, "text": "this um you know they want to get to a" }, { "startMs": 114560, "text": "this um you know they want to get to a world where they are deploying this" }, { "startMs": 116870, "text": "world where they are deploying this" }, { "startMs": 116880, "text": "world where they are deploying this transformational technology which is" }, { "startMs": 118709, "text": "transformational technology which is" }, { "startMs": 118719, "text": "transformational technology which is agents and LLMs to get the most possible" }, { "startMs": 121910, "text": "agents and LLMs to get the most possible" }, { "startMs": 121920, "text": "agents and LLMs to get the most possible business value at the lowest possible" }, { "startMs": 124389, "text": "business value at the lowest possible" }, { "startMs": 124399, "text": "business value at the lowest possible cost in a way that is governed and" }, { "startMs": 127429, "text": "cost in a way that is governed and" }, { "startMs": 127439, "text": "cost in a way that is governed and secure and they're running up against" }, { "startMs": 129589, "text": "secure and they're running up against" }, { "startMs": 129599, "text": "secure and they're running up against the limits of interactive agents which" }, { "startMs": 131350, "text": "the limits of interactive agents which" }, { "startMs": 131360, "text": "the limits of interactive agents which are like you you're bottlenecked by a" }, { "startMs": 134150, "text": "are like you you're bottlenecked by a" }, { "startMs": 134160, "text": "are like you you're bottlenecked by a person running them. Um it's hard to" }, { "startMs": 137190, "text": "person running them. Um it's hard to" }, { "startMs": 137200, "text": "person running them. Um it's hard to control costs if everyone has their own" }, { "startMs": 138790, "text": "control costs if everyone has their own" }, { "startMs": 138800, "text": "control costs if everyone has their own setup and it's hard to govern. And so" }, { "startMs": 141589, "text": "setup and it's hard to govern. And so" }, { "startMs": 141599, "text": "setup and it's hard to govern. And so just to double click on that I think for" }, { "startMs": 143350, "text": "just to double click on that I think for" }, { "startMs": 143360, "text": "just to double click on that I think for this audience like local agents have" }, { "startMs": 145750, "text": "this audience like local agents have" }, { "startMs": 145760, "text": "this audience like local agents have this security problem where uh if there" }, { "startMs": 149430, "text": "this security problem where uh if there" }, { "startMs": 149440, "text": "this security problem where uh if there is a problem or compromise of a local" }, { "startMs": 151910, "text": "is a problem or compromise of a local" }, { "startMs": 151920, "text": "is a problem or compromise of a local agent then it's basically has the" }, { "startMs": 154229, "text": "agent then it's basically has the" }, { "startMs": 154239, "text": "agent then it's basically has the ability to take over whatever the user" }, { "startMs": 156390, "text": "ability to take over whatever the user" }, { "startMs": 156400, "text": "ability to take over whatever the user who signed in on their laptop is. And so" }, { "startMs": 158630, "text": "who signed in on their laptop is. And so" }, { "startMs": 158640, "text": "who signed in on their laptop is. And so that's not a good thing. Um local agents" }, { "startMs": 161190, "text": "that's not a good thing. Um local agents" }, { "startMs": 161200, "text": "that's not a good thing. Um local agents are also hard to govern. And so right" }, { "startMs": 162710, "text": "are also hard to govern. And so right" }, { "startMs": 162720, "text": "are also hard to govern. And so right now we're in this situation where you" }, { "startMs": 165190, "text": "now we're in this situation where you" }, { "startMs": 165200, "text": "now we're in this situation where you know everyone is kind of configuring" }, { "startMs": 166550, "text": "know everyone is kind of configuring" }, { "startMs": 166560, "text": "know everyone is kind of configuring their own MCPs, their own skills, their" }, { "startMs": 168630, "text": "their own MCPs, their own skills, their" }, { "startMs": 168640, "text": "their own MCPs, their own skills, their own tools. And it's like you can't see" }, { "startMs": 170550, "text": "own tools. And it's like you can't see" }, { "startMs": 170560, "text": "own tools. And it's like you can't see what's going on. You can't audit it. And" }, { "startMs": 172630, "text": "what's going on. You can't audit it. And" }, { "startMs": 172640, "text": "what's going on. You can't audit it. And so the solution uh to both these" }, { "startMs": 175589, "text": "so the solution uh to both these" }, { "startMs": 175599, "text": "so the solution uh to both these problems is to sandbox your agents and" }, { "startMs": 178150, "text": "problems is to sandbox your agents and" }, { "startMs": 178160, "text": "problems is to sandbox your agents and move them to the cloud. And the product" }, { "startMs": 180710, "text": "move them to the cloud. And the product" }, { "startMs": 180720, "text": "move them to the cloud. And the product category that is emerging to support" }, { "startMs": 183030, "text": "category that is emerging to support" }, { "startMs": 183040, "text": "category that is emerging to support this uh is what um you know we call a" }, { "startMs": 185990, "text": "this uh is what um you know we call a" }, { "startMs": 186000, "text": "this uh is what um you know we call a cloud software factory. [snorts] And so" }, { "startMs": 188229, "text": "cloud software factory. [snorts] And so" }, { "startMs": 188239, "text": "cloud software factory. [snorts] And so this this term is kind of overloaded," }, { "startMs": 190229, "text": "this this term is kind of overloaded," }, { "startMs": 190239, "text": "this this term is kind of overloaded, but what it means to me is basically" }, { "startMs": 195030, "text": "but what it means to me is basically" }, { "startMs": 195040, "text": "but what it means to me is basically every company is going to set up a loop" }, { "startMs": 197350, "text": "every company is going to set up a loop" }, { "startMs": 197360, "text": "every company is going to set up a loop in the cloud where" }, { "startMs": 199830, "text": "in the cloud where" }, { "startMs": 199840, "text": "in the cloud where um a combination of humans and agents do" }, { "startMs": 203509, "text": "um a combination of humans and agents do" }, { "startMs": 203519, "text": "um a combination of humans and agents do work. And so humans are responsible for" }, { "startMs": 206630, "text": "work. And so humans are responsible for" }, { "startMs": 206640, "text": "work. And so humans are responsible for putting work in, agents will do" }, { "startMs": 209830, "text": "putting work in, agents will do" }, { "startMs": 209840, "text": "putting work in, agents will do different parts of the software life" }, { "startMs": 211270, "text": "different parts of the software life" }, { "startMs": 211280, "text": "different parts of the software life cycle. also a triage. They'll work with" }, { "startMs": 213270, "text": "cycle. also a triage. They'll work with" }, { "startMs": 213280, "text": "cycle. also a triage. They'll work with the human on a spec. They'll do" }, { "startMs": 215670, "text": "the human on a spec. They'll do" }, { "startMs": 215680, "text": "the human on a spec. They'll do implementation. They will do computer" }, { "startMs": 218070, "text": "implementation. They will do computer" }, { "startMs": 218080, "text": "implementation. They will do computer use verification. And as a as a" }, { "startMs": 221190, "text": "use verification. And as a as a" }, { "startMs": 221200, "text": "use verification. And as a as a business, what you're going to want to" }, { "startMs": 222470, "text": "business, what you're going to want to" }, { "startMs": 222480, "text": "business, what you're going to want to do is set these up internally. Define" }, { "startMs": 226229, "text": "do is set these up internally. Define" }, { "startMs": 226239, "text": "do is set these up internally. Define all the rules for how humans and agents" }, { "startMs": 227932, "text": "all the rules for how humans and agents" }, { "startMs": 227942, "text": "all the rules for how humans and agents [music]" }, { "startMs": 228630, "text": "[music]" }, { "startMs": 228640, "text": "[music] interact and then get to a spot where" }, { "startMs": 230869, "text": "interact and then get to a spot where" }, { "startMs": 230879, "text": "interact and then get to a spot where you can really, really measure and" }, { "startMs": 233110, "text": "you can really, really measure and" }, { "startMs": 233120, "text": "you can really, really measure and improve over time. Because getting this" }, { "startMs": 235589, "text": "improve over time. Because getting this" }, { "startMs": 235599, "text": "improve over time. Because getting this concept right is going to be the key" }, { "startMs": 238470, "text": "concept right is going to be the key" }, { "startMs": 238480, "text": "concept right is going to be the key driver to how your organization ships" }, { "startMs": 240390, "text": "driver to how your organization ships" }, { "startMs": 240400, "text": "driver to how your organization ships software." }, { "startMs": 242149, "text": "software." }, { "startMs": 242159, "text": "software. Um at at warp like our our belief is" }, { "startMs": 244949, "text": "Um at at warp like our our belief is" }, { "startMs": 244959, "text": "Um at at warp like our our belief is that you should think of this as" }, { "startMs": 246229, "text": "that you should think of this as" }, { "startMs": 246239, "text": "that you should think of this as infrastructure. So not as like a product" }, { "startMs": 248470, "text": "infrastructure. So not as like a product" }, { "startMs": 248480, "text": "infrastructure. So not as like a product or an AI teammate but really like an" }, { "startMs": 250949, "text": "or an AI teammate but really like an" }, { "startMs": 250959, "text": "or an AI teammate but really like an infrastructure stack that you can build" }, { "startMs": 253190, "text": "infrastructure stack that you can build" }, { "startMs": 253200, "text": "infrastructure stack that you can build on top of and scale. And the key aspects" }, { "startMs": 256469, "text": "on top of and scale. And the key aspects" }, { "startMs": 256479, "text": "on top of and scale. And the key aspects of this are like it should be open, it" }, { "startMs": 259110, "text": "of this are like it should be open, it" }, { "startMs": 259120, "text": "of this are like it should be open, it should be adaptable and you should think" }, { "startMs": 260710, "text": "should be adaptable and you should think" }, { "startMs": 260720, "text": "should be adaptable and you should think of your factories as something that you" }, { "startMs": 262550, "text": "of your factories as something that you" }, { "startMs": 262560, "text": "of your factories as something that you define in code. And so by open I mean" }, { "startMs": 267030, "text": "define in code. And so by open I mean" }, { "startMs": 267040, "text": "define in code. And so by open I mean you want a a factory model that can" }, { "startMs": 269350, "text": "you want a a factory model that can" }, { "startMs": 269360, "text": "you want a a factory model that can support different uh different models" }, { "startMs": 271749, "text": "support different uh different models" }, { "startMs": 271759, "text": "support different uh different models whether it's open weight or frontier." }, { "startMs": 273670, "text": "whether it's open weight or frontier." }, { "startMs": 273680, "text": "whether it's open weight or frontier. You want one that can support different" }, { "startMs": 275110, "text": "You want one that can support different" }, { "startMs": 275120, "text": "You want one that can support different harnesses and the reason is just like" }, { "startMs": 277909, "text": "harnesses and the reason is just like" }, { "startMs": 277919, "text": "harnesses and the reason is just like the best models and harnesses are" }, { "startMs": 279590, "text": "the best models and harnesses are" }, { "startMs": 279600, "text": "the best models and harnesses are constantly changing. You want adaptable" }, { "startMs": 282230, "text": "constantly changing. You want adaptable" }, { "startMs": 282240, "text": "constantly changing. You want adaptable means like the factory should work with" }, { "startMs": 283909, "text": "means like the factory should work with" }, { "startMs": 283919, "text": "means like the factory should work with your existing infrastructure. And then" }, { "startMs": 286150, "text": "your existing infrastructure. And then" }, { "startMs": 286160, "text": "your existing infrastructure. And then finally defined in code means like your" }, { "startMs": 289030, "text": "finally defined in code means like your" }, { "startMs": 289040, "text": "finally defined in code means like your factory should be a thing uh that has" }, { "startMs": 293270, "text": "factory should be a thing uh that has" }, { "startMs": 293280, "text": "factory should be a thing uh that has like a strict definition and that" }, { "startMs": 295030, "text": "like a strict definition and that" }, { "startMs": 295040, "text": "like a strict definition and that definition should include like what are" }, { "startMs": 297350, "text": "definition should include like what are" }, { "startMs": 297360, "text": "definition should include like what are the workflows, what are the skills, what" }, { "startMs": 299110, "text": "the workflows, what are the skills, what" }, { "startMs": 299120, "text": "the workflows, what are the skills, what are the MCPs, where do these run and" }, { "startMs": 301510, "text": "are the MCPs, where do these run and" }, { "startMs": 301520, "text": "are the MCPs, where do these run and when you define it in code and version" }, { "startMs": 303670, "text": "when you define it in code and version" }, { "startMs": 303680, "text": "when you define it in code and version control it then you get the ability to" }, { "startMs": 305749, "text": "control it then you get the ability to" }, { "startMs": 305759, "text": "control it then you get the ability to like test different versions of your" }, { "startMs": 307749, "text": "like test different versions of your" }, { "startMs": 307759, "text": "like test different versions of your factory. You get the ability for coding" }, { "startMs": 309350, "text": "factory. You get the ability for coding" }, { "startMs": 309360, "text": "factory. You get the ability for coding agents to update your factory. So these" }, { "startMs": 311830, "text": "agents to update your factory. So these" }, { "startMs": 311840, "text": "agents to update your factory. So these are some key characteristics." }, { "startMs": 314230, "text": "are some key characteristics." }, { "startMs": 314240, "text": "are some key characteristics. Uh I'll say another thing that I'm" }, { "startMs": 316150, "text": "Uh I'll say another thing that I'm" }, { "startMs": 316160, "text": "Uh I'll say another thing that I'm hearing from from businesses is like AI" }, { "startMs": 319189, "text": "hearing from from businesses is like AI" }, { "startMs": 319199, "text": "hearing from from businesses is like AI sovereignty matters. And so the concept" }, { "startMs": 321749, "text": "sovereignty matters. And so the concept" }, { "startMs": 321759, "text": "sovereignty matters. And so the concept here is like as you think about" }, { "startMs": 323990, "text": "here is like as you think about" }, { "startMs": 324000, "text": "here is like as you think about factories, you should think about uh" }, { "startMs": 326629, "text": "factories, you should think about uh" }, { "startMs": 326639, "text": "factories, you should think about uh owning all the data that goes in and all" }, { "startMs": 329350, "text": "owning all the data that goes in and all" }, { "startMs": 329360, "text": "owning all the data that goes in and all the data that comes out as well." }, { "startMs": 332310, "text": "the data that comes out as well." }, { "startMs": 332320, "text": "the data that comes out as well. You should think about how do you have" }, { "startMs": 333590, "text": "You should think about how do you have" }, { "startMs": 333600, "text": "You should think about how do you have these agents running on your own" }, { "startMs": 335590, "text": "these agents running on your own" }, { "startMs": 335600, "text": "these agents running on your own infrastructure and own compute. Um this" }, { "startMs": 337510, "text": "infrastructure and own compute. Um this" }, { "startMs": 337520, "text": "infrastructure and own compute. Um this is what most larger enterprises that we" }, { "startMs": 339830, "text": "is what most larger enterprises that we" }, { "startMs": 339840, "text": "is what most larger enterprises that we work with want." }, { "startMs": 341830, "text": "work with want." }, { "startMs": 341840, "text": "work with want. How do you provide your own inference?" }, { "startMs": 344390, "text": "How do you provide your own inference?" }, { "startMs": 344400, "text": "How do you provide your own inference? And then finally, like I said, any" }, { "startMs": 345909, "text": "And then finally, like I said, any" }, { "startMs": 345919, "text": "And then finally, like I said, any model, any harness. And so really" }, { "startMs": 348390, "text": "model, any harness. And so really" }, { "startMs": 348400, "text": "model, any harness. And so really preserve optionality and treat um treat" }, { "startMs": 351430, "text": "preserve optionality and treat um treat" }, { "startMs": 351440, "text": "preserve optionality and treat um treat these software factories as something" }, { "startMs": 352870, "text": "these software factories as something" }, { "startMs": 352880, "text": "these software factories as something that's very key to your business and" }, { "startMs": 355350, "text": "that's very key to your business and" }, { "startMs": 355360, "text": "that's very key to your business and that you want to, you know, mostly take" }, { "startMs": 357830, "text": "that you want to, you know, mostly take" }, { "startMs": 357840, "text": "that you want to, you know, mostly take ownership of the core ingredients of." }, { "startMs": 361189, "text": "ownership of the core ingredients of." }, { "startMs": 361199, "text": "ownership of the core ingredients of. Um, another really important point here" }, { "startMs": 363430, "text": "Um, another really important point here" }, { "startMs": 363440, "text": "Um, another really important point here is like as you think about building your" }, { "startMs": 366230, "text": "is like as you think about building your" }, { "startMs": 366240, "text": "is like as you think about building your software factory, like if you were to" }, { "startMs": 368710, "text": "software factory, like if you were to" }, { "startMs": 368720, "text": "software factory, like if you were to look on Twitter on any given day, you" }, { "startMs": 370550, "text": "look on Twitter on any given day, you" }, { "startMs": 370560, "text": "look on Twitter on any given day, you would probably see different model like" }, { "startMs": 373430, "text": "would probably see different model like" }, { "startMs": 373440, "text": "would probably see different model like someone would say like PI is the best or" }, { "startMs": 374950, "text": "someone would say like PI is the best or" }, { "startMs": 374960, "text": "someone would say like PI is the best or someone say Grock or like there's all" }, { "startMs": 376790, "text": "someone say Grock or like there's all" }, { "startMs": 376800, "text": "someone say Grock or like there's all these different things in the world" }, { "startMs": 378230, "text": "these different things in the world" }, { "startMs": 378240, "text": "these different things in the world around like what is best and you might" }, { "startMs": 379909, "text": "around like what is best and you might" }, { "startMs": 379919, "text": "around like what is best and you might even see benchmarks like Sweetbench or" }, { "startMs": 381590, "text": "even see benchmarks like Sweetbench or" }, { "startMs": 381600, "text": "even see benchmarks like Sweetbench or Terminal Bench. But the world that you" }, { "startMs": 383670, "text": "Terminal Bench. But the world that you" }, { "startMs": 383680, "text": "Terminal Bench. But the world that you want to get to um as you automate more" }, { "startMs": 386950, "text": "want to get to um as you automate more" }, { "startMs": 386960, "text": "want to get to um as you automate more development is actually being" }, { "startMs": 389110, "text": "development is actually being" }, { "startMs": 389120, "text": "development is actually being measurement first on your own data. And" }, { "startMs": 391510, "text": "measurement first on your own data. And" }, { "startMs": 391520, "text": "measurement first on your own data. And so you should whatever factory model you" }, { "startMs": 393749, "text": "so you should whatever factory model you" }, { "startMs": 393759, "text": "so you should whatever factory model you approach have the ability to score" }, { "startMs": 396629, "text": "approach have the ability to score" }, { "startMs": 396639, "text": "approach have the ability to score meaning like eval how your coding agents" }, { "startMs": 398550, "text": "meaning like eval how your coding agents" }, { "startMs": 398560, "text": "meaning like eval how your coding agents are doing. uh feed those scores into" }, { "startMs": 401909, "text": "are doing. uh feed those scores into" }, { "startMs": 401919, "text": "are doing. uh feed those scores into agents that can improve your factory and" }, { "startMs": 404309, "text": "agents that can improve your factory and" }, { "startMs": 404319, "text": "agents that can improve your factory and then also create your own benchmarks." }, { "startMs": 406550, "text": "then also create your own benchmarks." }, { "startMs": 406560, "text": "then also create your own benchmarks. Meaning like be able to take all of your" }, { "startMs": 408870, "text": "Meaning like be able to take all of your" }, { "startMs": 408880, "text": "Meaning like be able to take all of your real coding agent tasks and try them" }, { "startMs": 411270, "text": "real coding agent tasks and try them" }, { "startMs": 411280, "text": "real coding agent tasks and try them against different models and measure how" }, { "startMs": 412950, "text": "against different models and measure how" }, { "startMs": 412960, "text": "against different models and measure how it goes." }, { "startMs": 415749, "text": "it goes." }, { "startMs": 415759, "text": "it goes. So I'll also talk a little bit about" }, { "startMs": 417189, "text": "So I'll also talk a little bit about" }, { "startMs": 417199, "text": "So I'll also talk a little bit about like um how engineering changes in this" }, { "startMs": 421110, "text": "like um how engineering changes in this" }, { "startMs": 421120, "text": "like um how engineering changes in this factory world." }, { "startMs": 422479, "text": "[snorts] And so the way that I I conceive of this" }, { "startMs": 425270, "text": "And so the way that I I conceive of this" }, { "startMs": 425280, "text": "And so the way that I I conceive of this and talk about it is you will still have" }, { "startMs": 427110, "text": "and talk about it is you will still have" }, { "startMs": 427120, "text": "and talk about it is you will still have engineers who are very much responsible" }, { "startMs": 429670, "text": "engineers who are very much responsible" }, { "startMs": 429680, "text": "engineers who are very much responsible for building useful things for your" }, { "startMs": 432070, "text": "for building useful things for your" }, { "startMs": 432080, "text": "for building useful things for your customers. And so they'll build be" }, { "startMs": 433909, "text": "customers. And so they'll build be" }, { "startMs": 433919, "text": "customers. And so they'll build be building the features, they'll be fixing" }, { "startMs": 435189, "text": "building the features, they'll be fixing" }, { "startMs": 435199, "text": "building the features, they'll be fixing bugs, they'll be doing product" }, { "startMs": 436309, "text": "bugs, they'll be doing product" }, { "startMs": 436319, "text": "bugs, they'll be doing product engineering and they'll be doing it like" }, { "startMs": 438309, "text": "engineering and they'll be doing it like" }, { "startMs": 438319, "text": "engineering and they'll be doing it like through the factory. So they put work" }, { "startMs": 439990, "text": "through the factory. So they put work" }, { "startMs": 440000, "text": "through the factory. So they put work into the factory. But there's a second" }, { "startMs": 442390, "text": "into the factory. But there's a second" }, { "startMs": 442400, "text": "into the factory. But there's a second kind of engineering which I call factory" }, { "startMs": 445110, "text": "kind of engineering which I call factory" }, { "startMs": 445120, "text": "kind of engineering which I call factory engineering which is uh either uh your" }, { "startMs": 449510, "text": "engineering which is uh either uh your" }, { "startMs": 449520, "text": "engineering which is uh either uh your platform team or just your general" }, { "startMs": 451110, "text": "platform team or just your general" }, { "startMs": 451120, "text": "platform team or just your general engineers will have to get good at like" }, { "startMs": 454629, "text": "engineers will have to get good at like" }, { "startMs": 454639, "text": "engineers will have to get good at like trying to make the factory better. So" }, { "startMs": 457110, "text": "trying to make the factory better. So" }, { "startMs": 457120, "text": "trying to make the factory better. So improving the skills," }, { "startMs": 459430, "text": "improving the skills," }, { "startMs": 459440, "text": "improving the skills, setting the context, changing the model" }, { "startMs": 461830, "text": "setting the context, changing the model" }, { "startMs": 461840, "text": "setting the context, changing the model mix, all of these things are going to" }, { "startMs": 463430, "text": "mix, all of these things are going to" }, { "startMs": 463440, "text": "mix, all of these things are going to become more important as engineers are" }, { "startMs": 465749, "text": "become more important as engineers are" }, { "startMs": 465759, "text": "become more important as engineers are like building and improving the system" }, { "startMs": 468309, "text": "like building and improving the system" }, { "startMs": 468319, "text": "like building and improving the system that actually builds the software." }, { "startMs": 471510, "text": "that actually builds the software." }, { "startMs": 471520, "text": "that actually builds the software. So, I'm going to I'm going to show what" }, { "startMs": 473110, "text": "So, I'm going to I'm going to show what" }, { "startMs": 473120, "text": "So, I'm going to I'm going to show what some of this looks like in practice. And" }, { "startMs": 474790, "text": "some of this looks like in practice. And" }, { "startMs": 474800, "text": "some of this looks like in practice. And again, this is going to be in the" }, { "startMs": 476550, "text": "again, this is going to be in the" }, { "startMs": 476560, "text": "again, this is going to be in the context of our own product, Warp" }, { "startMs": 478070, "text": "context of our own product, Warp" }, { "startMs": 478080, "text": "context of our own product, Warp Factories. But these general concepts, I" }, { "startMs": 480950, "text": "Factories. But these general concepts, I" }, { "startMs": 480960, "text": "Factories. But these general concepts, I think, will translate to however you're" }, { "startMs": 483110, "text": "think, will translate to however you're" }, { "startMs": 483120, "text": "think, will translate to however you're thinking about building your factories." }, { "startMs": 486150, "text": "thinking about building your factories." }, { "startMs": 486160, "text": "thinking about building your factories. I'll try to leave a little bit of time" }, { "startMs": 487350, "text": "I'll try to leave a little bit of time" }, { "startMs": 487360, "text": "I'll try to leave a little bit of time for questions at the end if I can." }, { "startMs": 490070, "text": "for questions at the end if I can." }, { "startMs": 490080, "text": "for questions at the end if I can. So" }, { "startMs": 492629, "text": "So" }, { "startMs": 492639, "text": "So I'm going to start with um the view of" }, { "startMs": 496550, "text": "I'm going to start with um the view of" }, { "startMs": 496560, "text": "I'm going to start with um the view of the factory that the factory engineer or" }, { "startMs": 499670, "text": "the factory that the factory engineer or" }, { "startMs": 499680, "text": "the factory that the factory engineer or the manager has. And so this is like" }, { "startMs": 503510, "text": "the manager has. And so this is like" }, { "startMs": 503520, "text": "the manager has. And so this is like if you are" }, { "startMs": 505589, "text": "if you are" }, { "startMs": 505599, "text": "if you are someone who is trying to get better uh" }, { "startMs": 508629, "text": "someone who is trying to get better uh" }, { "startMs": 508639, "text": "someone who is trying to get better uh productivity across your team, you're" }, { "startMs": 510869, "text": "productivity across your team, you're" }, { "startMs": 510879, "text": "productivity across your team, you're going to want some view of like what is" }, { "startMs": 513990, "text": "going to want some view of like what is" }, { "startMs": 514000, "text": "going to want some view of like what is happening from a productivity standpoint" }, { "startMs": 515829, "text": "happening from a productivity standpoint" }, { "startMs": 515839, "text": "happening from a productivity standpoint like how automated is my factory? Uh" }, { "startMs": 518790, "text": "like how automated is my factory? Uh" }, { "startMs": 518800, "text": "like how automated is my factory? Uh you're going to want a view of um what's" }, { "startMs": 521430, "text": "you're going to want a view of um what's" }, { "startMs": 521440, "text": "you're going to want a view of um what's it what's the cycle time. So like a" }, { "startMs": 523269, "text": "it what's the cycle time. So like a" }, { "startMs": 523279, "text": "it what's the cycle time. So like a dorometric view. You're going to want a" }, { "startMs": 525509, "text": "dorometric view. You're going to want a" }, { "startMs": 525519, "text": "dorometric view. You're going to want a view of uh spending. This might take a" }, { "startMs": 527670, "text": "view of uh spending. This might take a" }, { "startMs": 527680, "text": "view of uh spending. This might take a second to load. Um but basically you" }, { "startMs": 530310, "text": "second to load. Um but basically you" }, { "startMs": 530320, "text": "second to load. Um but basically you want all this data tracked. And the way" }, { "startMs": 533350, "text": "want all this data tracked. And the way" }, { "startMs": 533360, "text": "want all this data tracked. And the way that you get these views is by having a" }, { "startMs": 537910, "text": "that you get these views is by having a" }, { "startMs": 537920, "text": "that you get these views is by having a system where everything that your team" }, { "startMs": 541509, "text": "system where everything that your team" }, { "startMs": 541519, "text": "system where everything that your team is doing, I don't know if you all can" }, { "startMs": 543829, "text": "is doing, I don't know if you all can" }, { "startMs": 543839, "text": "is doing, I don't know if you all can see this, but everything that's" }, { "startMs": 545590, "text": "see this, but everything that's" }, { "startMs": 545600, "text": "see this, but everything that's happening on our team, and this is" }, { "startMs": 547269, "text": "happening on our team, and this is" }, { "startMs": 547279, "text": "happening on our team, and this is Warp's internal software development" }, { "startMs": 549750, "text": "Warp's internal software development" }, { "startMs": 549760, "text": "Warp's internal software development that I'm leaking here, but it's like I" }, { "startMs": 552310, "text": "that I'm leaking here, but it's like I" }, { "startMs": 552320, "text": "that I'm leaking here, but it's like I can see what I can see what every" }, { "startMs": 554630, "text": "can see what I can see what every" }, { "startMs": 554640, "text": "can see what I can see what every engineer on our team is doing, and I can" }, { "startMs": 556470, "text": "engineer on our team is doing, and I can" }, { "startMs": 556480, "text": "engineer on our team is doing, and I can see what all the automations that we're" }, { "startMs": 557829, "text": "see what all the automations that we're" }, { "startMs": 557839, "text": "see what all the automations that we're running are doing. And not just can I" }, { "startMs": 560310, "text": "running are doing. And not just can I" }, { "startMs": 560320, "text": "running are doing. And not just can I see them like I can see all the" }, { "startMs": 561670, "text": "see them like I can see all the" }, { "startMs": 561680, "text": "see them like I can see all the artifacts that they're creating, but all" }, { "startMs": 563910, "text": "artifacts that they're creating, but all" }, { "startMs": 563920, "text": "artifacts that they're creating, but all this is getting stored. And so all these" }, { "startMs": 566230, "text": "this is getting stored. And so all these" }, { "startMs": 566240, "text": "this is getting stored. And so all these agent traces, you know, I I could" }, { "startMs": 568630, "text": "agent traces, you know, I I could" }, { "startMs": 568640, "text": "agent traces, you know, I I could actually go in I could see what this" }, { "startMs": 570790, "text": "actually go in I could see what this" }, { "startMs": 570800, "text": "actually go in I could see what this this one is doing. I could go in and" }, { "startMs": 572630, "text": "this one is doing. I could go in and" }, { "startMs": 572640, "text": "this one is doing. I could go in and like see the actual conversation that" }, { "startMs": 574470, "text": "like see the actual conversation that" }, { "startMs": 574480, "text": "like see the actual conversation that someone is doing. But the important" }, { "startMs": 576070, "text": "someone is doing. But the important" }, { "startMs": 576080, "text": "someone is doing. But the important thing here is that um all this data is" }, { "startMs": 579509, "text": "thing here is that um all this data is" }, { "startMs": 579519, "text": "thing here is that um all this data is getting stored and what your coding" }, { "startMs": 581750, "text": "getting stored and what your coding" }, { "startMs": 581760, "text": "getting stored and what your coding agents are doing is the raw material for" }, { "startMs": 584070, "text": "agents are doing is the raw material for" }, { "startMs": 584080, "text": "agents are doing is the raw material for improving them. And so, uh, you might be" }, { "startMs": 586949, "text": "improving them. And so, uh, you might be" }, { "startMs": 586959, "text": "improving them. And so, uh, you might be wondering, well, how do how do engineers" }, { "startMs": 589190, "text": "wondering, well, how do how do engineers" }, { "startMs": 589200, "text": "wondering, well, how do how do engineers actually do this work? Like, how are" }, { "startMs": 590630, "text": "actually do this work? Like, how are" }, { "startMs": 590640, "text": "actually do this work? Like, how are they putting stuff in? Um, and the" }, { "startMs": 592870, "text": "they putting stuff in? Um, and the" }, { "startMs": 592880, "text": "they putting stuff in? Um, and the answer on this is that they put it in" }, { "startMs": 595590, "text": "answer on this is that they put it in" }, { "startMs": 595600, "text": "answer on this is that they put it in through the tools that they're already" }, { "startMs": 596949, "text": "through the tools that they're already" }, { "startMs": 596959, "text": "through the tools that they're already using. So, on our team, they're using" }, { "startMs": 599350, "text": "using. So, on our team, they're using" }, { "startMs": 599360, "text": "using. So, on our team, they're using Slack uh to do a lot of the work. Um," }, { "startMs": 602070, "text": "Slack uh to do a lot of the work. Um," }, { "startMs": 602080, "text": "Slack uh to do a lot of the work. Um, they put in uh agentic work through our" }, { "startMs": 604870, "text": "they put in uh agentic work through our" }, { "startMs": 604880, "text": "they put in uh agentic work through our task tracker, which we use linear, but" }, { "startMs": 606550, "text": "task tracker, which we use linear, but" }, { "startMs": 606560, "text": "task tracker, which we use linear, but it could be Jira. They have the same" }, { "startMs": 608630, "text": "it could be Jira. They have the same" }, { "startMs": 608640, "text": "it could be Jira. They have the same context whether they're uh responding to" }, { "startMs": 611190, "text": "context whether they're uh responding to" }, { "startMs": 611200, "text": "context whether they're uh responding to a GitHub PR comment. And so the the idea" }, { "startMs": 614470, "text": "a GitHub PR comment. And so the the idea" }, { "startMs": 614480, "text": "a GitHub PR comment. And so the the idea here is like the product engineers are" }, { "startMs": 615990, "text": "here is like the product engineers are" }, { "startMs": 616000, "text": "here is like the product engineers are using the tools that they're already" }, { "startMs": 617269, "text": "using the tools that they're already" }, { "startMs": 617279, "text": "using the tools that they're already using, but all the work goes into this" }, { "startMs": 619350, "text": "using, but all the work goes into this" }, { "startMs": 619360, "text": "using, but all the work goes into this centralized location where you get a" }, { "startMs": 621030, "text": "centralized location where you get a" }, { "startMs": 621040, "text": "centralized location where you get a centralized view and all this stuff gets" }, { "startMs": 623190, "text": "centralized view and all this stuff gets" }, { "startMs": 623200, "text": "centralized view and all this stuff gets recorded over time by recording" }, { "startMs": 625829, "text": "recorded over time by recording" }, { "startMs": 625839, "text": "recorded over time by recording everything. Uh just to go back to this" }, { "startMs": 628790, "text": "everything. Uh just to go back to this" }, { "startMs": 628800, "text": "everything. Uh just to go back to this dashboard for a second. Um actually let" }, { "startMs": 632069, "text": "dashboard for a second. Um actually let" }, { "startMs": 632079, "text": "dashboard for a second. Um actually let me let me not do that for one second." }, { "startMs": 633430, "text": "me let me not do that for one second." }, { "startMs": 633440, "text": "me let me not do that for one second. The other thing that's important here" }, { "startMs": 634550, "text": "The other thing that's important here" }, { "startMs": 634560, "text": "The other thing that's important here that I I talked about earlier is like" }, { "startMs": 636870, "text": "that I I talked about earlier is like" }, { "startMs": 636880, "text": "that I I talked about earlier is like the factory workflow itself defining it" }, { "startMs": 639990, "text": "the factory workflow itself defining it" }, { "startMs": 640000, "text": "the factory workflow itself defining it as code. And so this is um our kind of" }, { "startMs": 643829, "text": "as code. And so this is um our kind of" }, { "startMs": 643839, "text": "as code. And so this is um our kind of like default factory workflow where" }, { "startMs": 646710, "text": "like default factory workflow where" }, { "startMs": 646720, "text": "like default factory workflow where think of it as like workflows in we have" }, { "startMs": 649430, "text": "think of it as like workflows in we have" }, { "startMs": 649440, "text": "think of it as like workflows in we have this orchestrator agent that will figure" }, { "startMs": 651910, "text": "this orchestrator agent that will figure" }, { "startMs": 651920, "text": "this orchestrator agent that will figure out okay should this next go to a triage" }, { "startMs": 654069, "text": "out okay should this next go to a triage" }, { "startMs": 654079, "text": "out okay should this next go to a triage step should it go to a spec step an" }, { "startMs": 655750, "text": "step should it go to a spec step an" }, { "startMs": 655760, "text": "step should it go to a spec step an implementation step code review the" }, { "startMs": 658470, "text": "implementation step code review the" }, { "startMs": 658480, "text": "implementation step code review the actual details of this need to be very" }, { "startMs": 660310, "text": "actual details of this need to be very" }, { "startMs": 660320, "text": "actual details of this need to be very very flexible. Um if you click into one" }, { "startMs": 663350, "text": "very flexible. Um if you click into one" }, { "startMs": 663360, "text": "very flexible. Um if you click into one of these you can see like essentially" }, { "startMs": 666550, "text": "of these you can see like essentially" }, { "startMs": 666560, "text": "of these you can see like essentially you are providing an agent definition" }, { "startMs": 669509, "text": "you are providing an agent definition" }, { "startMs": 669519, "text": "you are providing an agent definition there's uh which is like what are all" }, { "startMs": 671590, "text": "there's uh which is like what are all" }, { "startMs": 671600, "text": "there's uh which is like what are all the secrets the MCPS the agent prompt" }, { "startMs": 674230, "text": "the secrets the MCPS the agent prompt" }, { "startMs": 674240, "text": "the secrets the MCPS the agent prompt the model all this stuff the the cool" }, { "startMs": 677430, "text": "the model all this stuff the the cool" }, { "startMs": 677440, "text": "the model all this stuff the the cool thing though that I would emphasize that" }, { "startMs": 680389, "text": "thing though that I would emphasize that" }, { "startMs": 680399, "text": "thing though that I would emphasize that however you're approaching this you" }, { "startMs": 681910, "text": "however you're approaching this you" }, { "startMs": 681920, "text": "however you're approaching this you probably want to do is that there's an" }, { "startMs": 684069, "text": "probably want to do is that there's an" }, { "startMs": 684079, "text": "probably want to do is that there's an actual code repo that sits behind this" }, { "startMs": 687269, "text": "actual code repo that sits behind this" }, { "startMs": 687279, "text": "actual code repo that sits behind this where if you go and like look in that" }, { "startMs": 689269, "text": "where if you go and like look in that" }, { "startMs": 689279, "text": "where if you go and like look in that repo you can see like I that earlier, a" }, { "startMs": 692230, "text": "repo you can see like I that earlier, a" }, { "startMs": 692240, "text": "repo you can see like I that earlier, a definition of the entire factory at any" }, { "startMs": 695269, "text": "definition of the entire factory at any" }, { "startMs": 695279, "text": "definition of the entire factory at any given point in time. And so that" }, { "startMs": 696949, "text": "given point in time. And so that" }, { "startMs": 696959, "text": "given point in time. And so that definition includes, think of it as like" }, { "startMs": 699750, "text": "definition includes, think of it as like" }, { "startMs": 699760, "text": "definition includes, think of it as like everything that goes into the process of" }, { "startMs": 701430, "text": "everything that goes into the process of" }, { "startMs": 701440, "text": "everything that goes into the process of automating development. Um, it's your" }, { "startMs": 704230, "text": "automating development. Um, it's your" }, { "startMs": 704240, "text": "automating development. Um, it's your it's your uh agent definitions, it's" }, { "startMs": 707430, "text": "it's your uh agent definitions, it's" }, { "startMs": 707440, "text": "it's your uh agent definitions, it's your automations, your benchmarks, it's" }, { "startMs": 709030, "text": "your automations, your benchmarks, it's" }, { "startMs": 709040, "text": "your automations, your benchmarks, it's where these agents run, the skills, the" }, { "startMs": 710550, "text": "where these agents run, the skills, the" }, { "startMs": 710560, "text": "where these agents run, the skills, the web hooks, everything. So everything" }, { "startMs": 712150, "text": "web hooks, everything. So everything" }, { "startMs": 712160, "text": "web hooks, everything. So everything gets stored in in a repo that it allows" }, { "startMs": 715190, "text": "gets stored in in a repo that it allows" }, { "startMs": 715200, "text": "gets stored in in a repo that it allows for automation. Um, and I'll I'll I'll" }, { "startMs": 718949, "text": "for automation. Um, and I'll I'll I'll" }, { "startMs": 718959, "text": "for automation. Um, and I'll I'll I'll show a little more why that matters in a" }, { "startMs": 720550, "text": "show a little more why that matters in a" }, { "startMs": 720560, "text": "show a little more why that matters in a second." }, { "startMs": 721350, "text": "second." }, { "startMs": 721360, "text": "second. The other thing uh just to return to" }, { "startMs": 723670, "text": "The other thing uh just to return to" }, { "startMs": 723680, "text": "The other thing uh just to return to this view for a second because I think" }, { "startMs": 724949, "text": "this view for a second because I think" }, { "startMs": 724959, "text": "this view for a second because I think it's interesting. You get these metrics" }, { "startMs": 727190, "text": "it's interesting. You get these metrics" }, { "startMs": 727200, "text": "it's interesting. You get these metrics around productivity and spend but you" }, { "startMs": 729509, "text": "around productivity and spend but you" }, { "startMs": 729519, "text": "around productivity and spend but you also want to set up something where" }, { "startMs": 731750, "text": "also want to set up something where" }, { "startMs": 731760, "text": "also want to set up something where there are like we call them scores. You" }, { "startMs": 734310, "text": "there are like we call them scores. You" }, { "startMs": 734320, "text": "there are like we call them scores. You could think of it as like online evals." }, { "startMs": 737190, "text": "could think of it as like online evals." }, { "startMs": 737200, "text": "could think of it as like online evals. And what what you're looking at here is" }, { "startMs": 740069, "text": "And what what you're looking at here is" }, { "startMs": 740079, "text": "And what what you're looking at here is basically we have a set of agents that" }, { "startMs": 743430, "text": "basically we have a set of agents that" }, { "startMs": 743440, "text": "basically we have a set of agents that are looking at all of these runs that" }, { "startMs": 745829, "text": "are looking at all of these runs that" }, { "startMs": 745839, "text": "are looking at all of these runs that are going through our system. So" }, { "startMs": 747509, "text": "are going through our system. So" }, { "startMs": 747519, "text": "are going through our system. So remember I go back here. It's like you" }, { "startMs": 748870, "text": "remember I go back here. It's like you" }, { "startMs": 748880, "text": "remember I go back here. It's like you can see everything that everyone's doing" }, { "startMs": 750069, "text": "can see everything that everyone's doing" }, { "startMs": 750079, "text": "can see everything that everyone's doing on our team. You then have a scoring" }, { "startMs": 752790, "text": "on our team. You then have a scoring" }, { "startMs": 752800, "text": "on our team. You then have a scoring mechanism that goes and looks at those" }, { "startMs": 755670, "text": "mechanism that goes and looks at those" }, { "startMs": 755680, "text": "mechanism that goes and looks at those runs and sort of figures out by using" }, { "startMs": 759110, "text": "runs and sort of figures out by using" }, { "startMs": 759120, "text": "runs and sort of figures out by using LLM as a judge whether or not um those" }, { "startMs": 763590, "text": "LLM as a judge whether or not um those" }, { "startMs": 763600, "text": "LLM as a judge whether or not um those runs are good or not. And so for" }, { "startMs": 765990, "text": "runs are good or not. And so for" }, { "startMs": 766000, "text": "runs are good or not. And so for instance we have a scorer here that is" }, { "startMs": 769030, "text": "instance we have a scorer here that is" }, { "startMs": 769040, "text": "instance we have a scorer here that is looking for agents that are writing" }, { "startMs": 770230, "text": "looking for agents that are writing" }, { "startMs": 770240, "text": "looking for agents that are writing redundant tests which is a common" }, { "startMs": 772470, "text": "redundant tests which is a common" }, { "startMs": 772480, "text": "redundant tests which is a common failure mode with these coding agents." }, { "startMs": 774550, "text": "failure mode with these coding agents." }, { "startMs": 774560, "text": "failure mode with these coding agents. And so for the most part, our our agents" }, { "startMs": 776629, "text": "And so for the most part, our our agents" }, { "startMs": 776639, "text": "And so for the most part, our our agents seem to be doing pretty well here. But" }, { "startMs": 778310, "text": "seem to be doing pretty well here. But" }, { "startMs": 778320, "text": "seem to be doing pretty well here. But if you look back in time, you'll see" }, { "startMs": 779590, "text": "if you look back in time, you'll see" }, { "startMs": 779600, "text": "if you look back in time, you'll see there's some set of them where an agent" }, { "startMs": 781750, "text": "there's some set of them where an agent" }, { "startMs": 781760, "text": "there's some set of them where an agent is deciding, okay, this was this thing" }, { "startMs": 784150, "text": "is deciding, okay, this was this thing" }, { "startMs": 784160, "text": "is deciding, okay, this was this thing wrote tests that didn't need to be" }, { "startMs": 785590, "text": "wrote tests that didn't need to be" }, { "startMs": 785600, "text": "wrote tests that didn't need to be written. And so every one of these" }, { "startMs": 787910, "text": "written. And so every one of these" }, { "startMs": 787920, "text": "written. And so every one of these failures is an opportunity for improving" }, { "startMs": 791670, "text": "failures is an opportunity for improving" }, { "startMs": 791680, "text": "failures is an opportunity for improving how your system works. And so as you" }, { "startMs": 794710, "text": "how your system works. And so as you" }, { "startMs": 794720, "text": "how your system works. And so as you gather this information, and you know," }, { "startMs": 797110, "text": "gather this information, and you know," }, { "startMs": 797120, "text": "gather this information, and you know, you could go look at this as a person" }, { "startMs": 798790, "text": "you could go look at this as a person" }, { "startMs": 798800, "text": "you could go look at this as a person and grade it, but it's much more" }, { "startMs": 799990, "text": "and grade it, but it's much more" }, { "startMs": 800000, "text": "and grade it, but it's much more efficient to have agents do it. As you" }, { "startMs": 802230, "text": "efficient to have agents do it. As you" }, { "startMs": 802240, "text": "efficient to have agents do it. As you gather this information, you can then" }, { "startMs": 804310, "text": "gather this information, you can then" }, { "startMs": 804320, "text": "gather this information, you can then use it to feed what we call a" }, { "startMs": 806949, "text": "use it to feed what we call a" }, { "startMs": 806959, "text": "use it to feed what we call a self-improvement loop. And so the idea" }, { "startMs": 809670, "text": "self-improvement loop. And so the idea" }, { "startMs": 809680, "text": "self-improvement loop. And so the idea behind a self-improvement loop is that" }, { "startMs": 812550, "text": "behind a self-improvement loop is that" }, { "startMs": 812560, "text": "behind a self-improvement loop is that uh agents will go and they'll look at" }, { "startMs": 814150, "text": "uh agents will go and they'll look at" }, { "startMs": 814160, "text": "uh agents will go and they'll look at all these failure modes. They'll" }, { "startMs": 816310, "text": "all these failure modes. They'll" }, { "startMs": 816320, "text": "all these failure modes. They'll aggregate them. They will look for" }, { "startMs": 818629, "text": "aggregate them. They will look for" }, { "startMs": 818639, "text": "aggregate them. They will look for patterns and then they will come up with" }, { "startMs": 820949, "text": "patterns and then they will come up with" }, { "startMs": 820959, "text": "patterns and then they will come up with ideas for how do you improve the" }, { "startMs": 823829, "text": "ideas for how do you improve the" }, { "startMs": 823839, "text": "ideas for how do you improve the underlying factory definition so that" }, { "startMs": 826310, "text": "underlying factory definition so that" }, { "startMs": 826320, "text": "underlying factory definition so that going forward you don't have this same" }, { "startMs": 828230, "text": "going forward you don't have this same" }, { "startMs": 828240, "text": "going forward you don't have this same failure mode. And so you know for" }, { "startMs": 831990, "text": "failure mode. And so you know for" }, { "startMs": 832000, "text": "failure mode. And so you know for instance um I don't know I'm going to" }, { "startMs": 833750, "text": "instance um I don't know I'm going to" }, { "startMs": 833760, "text": "instance um I don't know I'm going to pick a random one here. Um this is like" }, { "startMs": 838629, "text": "pick a random one here. Um this is like" }, { "startMs": 838639, "text": "pick a random one here. Um this is like read matching repository skills before" }, { "startMs": 840629, "text": "read matching repository skills before" }, { "startMs": 840639, "text": "read matching repository skills before implementing. The key thing here is like" }, { "startMs": 843269, "text": "implementing. The key thing here is like" }, { "startMs": 843279, "text": "implementing. The key thing here is like it's saying I saw this fail 37 times and" }, { "startMs": 847350, "text": "it's saying I saw this fail 37 times and" }, { "startMs": 847360, "text": "it's saying I saw this fail 37 times and then it's saying okay we can actually um" }, { "startMs": 851750, "text": "then it's saying okay we can actually um" }, { "startMs": 851760, "text": "then it's saying okay we can actually um oops we can actually fix this" }, { "startMs": 855269, "text": "oops we can actually fix this" }, { "startMs": 855279, "text": "oops we can actually fix this um with a PR" }, { "startMs": 858790, "text": "um with a PR" }, { "startMs": 858800, "text": "um with a PR to the factory definition." }, { "startMs": 861509, "text": "to the factory definition." }, { "startMs": 861519, "text": "to the factory definition. So this is like closing the loop where" }, { "startMs": 864310, "text": "So this is like closing the loop where" }, { "startMs": 864320, "text": "So this is like closing the loop where the agent has identified a problem then" }, { "startMs": 865990, "text": "the agent has identified a problem then" }, { "startMs": 866000, "text": "the agent has identified a problem then it's suggesting a change to actually" }, { "startMs": 868310, "text": "it's suggesting a change to actually" }, { "startMs": 868320, "text": "it's suggesting a change to actually drive a better result in the factory. So" }, { "startMs": 870389, "text": "drive a better result in the factory. So" }, { "startMs": 870399, "text": "drive a better result in the factory. So this is like a a core concept I think as" }, { "startMs": 873350, "text": "this is like a a core concept I think as" }, { "startMs": 873360, "text": "this is like a a core concept I think as you think of like how do you really" }, { "startMs": 874790, "text": "you think of like how do you really" }, { "startMs": 874800, "text": "you think of like how do you really improve development it's it's about" }, { "startMs": 877509, "text": "improve development it's it's about" }, { "startMs": 877519, "text": "improve development it's it's about creating these loops where agents can" }, { "startMs": 879430, "text": "creating these loops where agents can" }, { "startMs": 879440, "text": "creating these loops where agents can identify problems and then fix them." }, { "startMs": 881990, "text": "identify problems and then fix them." }, { "startMs": 882000, "text": "identify problems and then fix them. There's one more thing that I want to" }, { "startMs": 883430, "text": "There's one more thing that I want to" }, { "startMs": 883440, "text": "There's one more thing that I want to show and then maybe I'll take a couple" }, { "startMs": 885350, "text": "show and then maybe I'll take a couple" }, { "startMs": 885360, "text": "show and then maybe I'll take a couple questions. Um I mentioned early on like" }, { "startMs": 889910, "text": "questions. Um I mentioned early on like" }, { "startMs": 889920, "text": "questions. Um I mentioned early on like you can't really trust the public" }, { "startMs": 891990, "text": "you can't really trust the public" }, { "startMs": 892000, "text": "you can't really trust the public benchmarks [clears throat]" }, { "startMs": 893350, "text": "benchmarks [clears throat]" }, { "startMs": 893360, "text": "benchmarks [clears throat] as like a way of understanding how good" }, { "startMs": 895430, "text": "as like a way of understanding how good" }, { "startMs": 895440, "text": "as like a way of understanding how good a certain model or harness is going to" }, { "startMs": 897590, "text": "a certain model or harness is going to" }, { "startMs": 897600, "text": "a certain model or harness is going to be on your own workflows. And so what" }, { "startMs": 899829, "text": "be on your own workflows. And so what" }, { "startMs": 899839, "text": "be on your own workflows. And so what you want to get to a point where you can" }, { "startMs": 901590, "text": "you want to get to a point where you can" }, { "startMs": 901600, "text": "you want to get to a point where you can do is basically take any past run. So we" }, { "startMs": 906870, "text": "do is basically take any past run. So we" }, { "startMs": 906880, "text": "do is basically take any past run. So we can do this like with any anything that" }, { "startMs": 909189, "text": "can do this like with any anything that" }, { "startMs": 909199, "text": "can do this like with any anything that we've done in our system. I could go" }, { "startMs": 910710, "text": "we've done in our system. I could go" }, { "startMs": 910720, "text": "we've done in our system. I could go into it here and I have the um" }, { "startMs": 914629, "text": "into it here and I have the um" }, { "startMs": 914639, "text": "into it here and I have the um sorry I picked a bad example. I um" }, { "startMs": 920880, "text": "uh maybe I'm picking a bad example. The" }, { "startMs": 922550, "text": "uh maybe I'm picking a bad example. The" }, { "startMs": 922560, "text": "uh maybe I'm picking a bad example. The the idea here is like you can basically" }, { "startMs": 924550, "text": "the idea here is like you can basically" }, { "startMs": 924560, "text": "the idea here is like you can basically take past runs. I don't know why I can't" }, { "startMs": 928470, "text": "take past runs. I don't know why I can't" }, { "startMs": 928480, "text": "take past runs. I don't know why I can't do this here and say okay I want to add" }, { "startMs": 930870, "text": "do this here and say okay I want to add" }, { "startMs": 930880, "text": "do this here and say okay I want to add this to a benchmark suite and what this" }, { "startMs": 933590, "text": "this to a benchmark suite and what this" }, { "startMs": 933600, "text": "this to a benchmark suite and what this is doing is say okay this is like a" }, { "startMs": 935110, "text": "is doing is say okay this is like a" }, { "startMs": 935120, "text": "is doing is say okay this is like a representative thing that I did in my" }, { "startMs": 936710, "text": "representative thing that I did in my" }, { "startMs": 936720, "text": "representative thing that I did in my factory that could be useful to measure" }, { "startMs": 940069, "text": "factory that could be useful to measure" }, { "startMs": 940079, "text": "factory that could be useful to measure by testing it with different model" }, { "startMs": 941910, "text": "by testing it with different model" }, { "startMs": 941920, "text": "by testing it with different model configurations and so think of like" }, { "startMs": 943910, "text": "configurations and so think of like" }, { "startMs": 943920, "text": "configurations and so think of like building up a a suite of like front-end" }, { "startMs": 945910, "text": "building up a a suite of like front-end" }, { "startMs": 945920, "text": "building up a a suite of like front-end tasks and the question that you're" }, { "startMs": 947670, "text": "tasks and the question that you're" }, { "startMs": 947680, "text": "tasks and the question that you're trying to answer is like what's the best" }, { "startMs": 949990, "text": "trying to answer is like what's the best" }, { "startMs": 950000, "text": "trying to answer is like what's the best model for me to use for front-end tasks" }, { "startMs": 951910, "text": "model for me to use for front-end tasks" }, { "startMs": 951920, "text": "model for me to use for front-end tasks and like you might look on Twitter but" }, { "startMs": 953590, "text": "and like you might look on Twitter but" }, { "startMs": 953600, "text": "and like you might look on Twitter but you might also just be like let" }, { "startMs": 955829, "text": "you might also just be like let" }, { "startMs": 955839, "text": "you might also just be like let actually test. And so what you can do or" }, { "startMs": 959910, "text": "actually test. And so what you can do or" }, { "startMs": 959920, "text": "actually test. And so what you can do or should try to set up for your own system" }, { "startMs": 962310, "text": "should try to set up for your own system" }, { "startMs": 962320, "text": "should try to set up for your own system and this is built into warp factory. So" }, { "startMs": 964069, "text": "and this is built into warp factory. So" }, { "startMs": 964079, "text": "and this is built into warp factory. So I'll show it here is basically curate" }, { "startMs": 967189, "text": "I'll show it here is basically curate" }, { "startMs": 967199, "text": "I'll show it here is basically curate one of these benchmarks and what what it" }, { "startMs": 971110, "text": "one of these benchmarks and what what it" }, { "startMs": 971120, "text": "one of these benchmarks and what what it does is it will replay your real past" }, { "startMs": 975110, "text": "does is it will replay your real past" }, { "startMs": 975120, "text": "does is it will replay your real past coding agent tasks and then generate for" }, { "startMs": 978710, "text": "coding agent tasks and then generate for" }, { "startMs": 978720, "text": "coding agent tasks and then generate for you what looks like a basically a cost" }, { "startMs": 981670, "text": "you what looks like a basically a cost" }, { "startMs": 981680, "text": "you what looks like a basically a cost versus quality trade-off graph. And so" }, { "startMs": 984310, "text": "versus quality trade-off graph. And so" }, { "startMs": 984320, "text": "versus quality trade-off graph. And so this is something you'll see when um you" }, { "startMs": 986790, "text": "this is something you'll see when um you" }, { "startMs": 986800, "text": "this is something you'll see when um you know the public benchmarks come out, but" }, { "startMs": 988310, "text": "know the public benchmarks come out, but" }, { "startMs": 988320, "text": "know the public benchmarks come out, but this is on your own data. And so in in" }, { "startMs": 991509, "text": "this is on your own data. And so in in" }, { "startMs": 991519, "text": "this is on your own data. And so in in this case, for example, like we could" }, { "startMs": 993189, "text": "this case, for example, like we could" }, { "startMs": 993199, "text": "this case, for example, like we could see GLM53 flash uh was 93% correct on" }, { "startMs": 998949, "text": "see GLM53 flash uh was 93% correct on" }, { "startMs": 998959, "text": "see GLM53 flash uh was 93% correct on these tasks at a cost of $2. GPT56 soul" }, { "startMs": 1002710, "text": "these tasks at a cost of $2. GPT56 soul" }, { "startMs": 1002720, "text": "these tasks at a cost of $2. GPT56 soul was like 95% correct, but it cost three" }, { "startMs": 1005509, "text": "was like 95% correct, but it cost three" }, { "startMs": 1005519, "text": "was like 95% correct, but it cost three times as much. And so you can take this" }, { "startMs": 1008150, "text": "times as much. And so you can take this" }, { "startMs": 1008160, "text": "times as much. And so you can take this information and then build a model" }, { "startMs": 1009829, "text": "information and then build a model" }, { "startMs": 1009839, "text": "information and then build a model routing strategy that you have real" }, { "startMs": 1012550, "text": "routing strategy that you have real" }, { "startMs": 1012560, "text": "routing strategy that you have real confidence is not just a generic one but" }, { "startMs": 1016069, "text": "confidence is not just a generic one but" }, { "startMs": 1016079, "text": "confidence is not just a generic one but for your actual code your stack is" }, { "startMs": 1019749, "text": "for your actual code your stack is" }, { "startMs": 1019759, "text": "for your actual code your stack is actually providing value. And so um this" }, { "startMs": 1022710, "text": "actually providing value. And so um this" }, { "startMs": 1022720, "text": "actually providing value. And so um this is like I think a good example of like" }, { "startMs": 1026710, "text": "is like I think a good example of like" }, { "startMs": 1026720, "text": "is like I think a good example of like closing the loop. And then just to go" }, { "startMs": 1028949, "text": "closing the loop. And then just to go" }, { "startMs": 1028959, "text": "closing the loop. And then just to go back I'm going to finish up here in a" }, { "startMs": 1030870, "text": "back I'm going to finish up here in a" }, { "startMs": 1030880, "text": "back I'm going to finish up here in a second." }, { "startMs": 1032789, "text": "second." }, { "startMs": 1032799, "text": "second. The" }, { "startMs": 1035189, "text": "The" }, { "startMs": 1035199, "text": "The the sort of like way that we think about" }, { "startMs": 1037590, "text": "the sort of like way that we think about" }, { "startMs": 1037600, "text": "the sort of like way that we think about this is like at the bottom of your" }, { "startMs": 1039750, "text": "this is like at the bottom of your" }, { "startMs": 1039760, "text": "this is like at the bottom of your stack, it's defined in code. It's think" }, { "startMs": 1042309, "text": "stack, it's defined in code. It's think" }, { "startMs": 1042319, "text": "stack, it's defined in code. It's think of a data layer. What's the data going" }, { "startMs": 1044150, "text": "of a data layer. What's the data going" }, { "startMs": 1044160, "text": "of a data layer. What's the data going in and out? Where do these agents run?" }, { "startMs": 1046630, "text": "in and out? Where do these agents run?" }, { "startMs": 1046640, "text": "in and out? Where do these agents run? You really want to sandbox that. You" }, { "startMs": 1049430, "text": "You really want to sandbox that. You" }, { "startMs": 1049440, "text": "You really want to sandbox that. You want inference that's flexible. You want" }, { "startMs": 1052230, "text": "want inference that's flexible. You want" }, { "startMs": 1052240, "text": "want inference that's flexible. You want a self-improvement layer that gives you" }, { "startMs": 1054630, "text": "a self-improvement layer that gives you" }, { "startMs": 1054640, "text": "a self-improvement layer that gives you something like benchmark scoring. You" }, { "startMs": 1056950, "text": "something like benchmark scoring. You" }, { "startMs": 1056960, "text": "something like benchmark scoring. You want orchestration so you can have" }, { "startMs": 1058390, "text": "want orchestration so you can have" }, { "startMs": 1058400, "text": "want orchestration so you can have multi-step workflows. And then you want" }, { "startMs": 1060070, "text": "multi-step workflows. And then you want" }, { "startMs": 1060080, "text": "multi-step workflows. And then you want access from all these existing tools." }, { "startMs": 1062310, "text": "access from all these existing tools." }, { "startMs": 1062320, "text": "access from all these existing tools. And so that's like the infrastructure" }, { "startMs": 1064630, "text": "And so that's like the infrastructure" }, { "startMs": 1064640, "text": "And so that's like the infrastructure that I think is going to it's going to" }, { "startMs": 1066710, "text": "that I think is going to it's going to" }, { "startMs": 1066720, "text": "that I think is going to it's going to look something like this. Um no matter" }, { "startMs": 1069669, "text": "look something like this. Um no matter" }, { "startMs": 1069679, "text": "look something like this. Um no matter how you approach this problem uh if" }, { "startMs": 1072070, "text": "how you approach this problem uh if" }, { "startMs": 1072080, "text": "how you approach this problem uh if you're interested in our approach" }, { "startMs": 1073110, "text": "you're interested in our approach" }, { "startMs": 1073120, "text": "you're interested in our approach obviously we're we're we would love to" }, { "startMs": 1075029, "text": "obviously we're we're we would love to" }, { "startMs": 1075039, "text": "obviously we're we're we would love to talk to you. But let me pause there." }, { "startMs": 1078070, "text": "talk to you. But let me pause there." }, { "startMs": 1078080, "text": "talk to you. But let me pause there. Hopefully this was interesting to folks." }, { "startMs": 1079830, "text": "Hopefully this was interesting to folks." }, { "startMs": 1079840, "text": "Hopefully this was interesting to folks. And I might have time for one question." }, { "startMs": 1082390, "text": "And I might have time for one question." }, { "startMs": 1082400, "text": "And I might have time for one question. I don't know. Or I might have to go." }, { "startMs": 1086070, "text": "I don't know. Or I might have to go." }, { "startMs": 1086080, "text": "I don't know. Or I might have to go. Yes. What what uh what advice do you" }, { "startMs": 1088630, "text": "Yes. What what uh what advice do you" }, { "startMs": 1088640, "text": "Yes. What what uh what advice do you have for testing and verification of" }, { "startMs": 1091110, "text": "have for testing and verification of" }, { "startMs": 1091120, "text": "have for testing and verification of code at the factory?" }, { "startMs": 1092789, "text": "code at the factory?" }, { "startMs": 1092799, "text": "code at the factory? Awesome question. So the you know one" }, { "startMs": 1095750, "text": "Awesome question. So the you know one" }, { "startMs": 1095760, "text": "Awesome question. So the you know one thing that could happen if you do this" }, { "startMs": 1097510, "text": "thing that could happen if you do this" }, { "startMs": 1097520, "text": "thing that could happen if you do this approach is you get like a pile up of" }, { "startMs": 1099270, "text": "approach is you get like a pile up of" }, { "startMs": 1099280, "text": "approach is you get like a pile up of PRs right and so you want some way of" }, { "startMs": 1101190, "text": "PRs right and so you want some way of" }, { "startMs": 1101200, "text": "PRs right and so you want some way of like getting them through the big things" }, { "startMs": 1103350, "text": "like getting them through the big things" }, { "startMs": 1103360, "text": "like getting them through the big things that I advise are spec driven" }, { "startMs": 1105430, "text": "that I advise are spec driven" }, { "startMs": 1105440, "text": "that I advise are spec driven development. So you can actually test um" }, { "startMs": 1108549, "text": "development. So you can actually test um" }, { "startMs": 1108559, "text": "development. So you can actually test um make sure that the changes are adherent" }, { "startMs": 1110710, "text": "make sure that the changes are adherent" }, { "startMs": 1110720, "text": "make sure that the changes are adherent to a spec. And the other thing is" }, { "startMs": 1112630, "text": "to a spec. And the other thing is" }, { "startMs": 1112640, "text": "to a spec. And the other thing is computer use verification which I didn't" }, { "startMs": 1114630, "text": "computer use verification which I didn't" }, { "startMs": 1114640, "text": "computer use verification which I didn't show but actually um you know if you can" }, { "startMs": 1117430, "text": "show but actually um you know if you can" }, { "startMs": 1117440, "text": "show but actually um you know if you can have agents that verify or produce" }, { "startMs": 1120230, "text": "have agents that verify or produce" }, { "startMs": 1120240, "text": "have agents that verify or produce evidence from the user standpoint that" }, { "startMs": 1121990, "text": "evidence from the user standpoint that" }, { "startMs": 1122000, "text": "evidence from the user standpoint that the thing works that can go a long way" }, { "startMs": 1124549, "text": "the thing works that can go a long way" }, { "startMs": 1124559, "text": "the thing works that can go a long way in terms of like lessening the code" }, { "startMs": 1126150, "text": "in terms of like lessening the code" }, { "startMs": 1126160, "text": "in terms of like lessening the code review burden. So those are the those" }, { "startMs": 1127909, "text": "review burden. So those are the those" }, { "startMs": 1127919, "text": "review burden. So those are the those are the two things that I recommend." }, { "startMs": 1130549, "text": "are the two things that I recommend." }, { "startMs": 1130559, "text": "are the two things that I recommend. Yes." }, { "startMs": 1130950, "text": "Yes." }, { "startMs": 1130960, "text": "Yes. So how do you plan um the budgets when" }, { "startMs": 1133510, "text": "So how do you plan um the budgets when" }, { "startMs": 1133520, "text": "So how do you plan um the budgets when we are starting a project? How do you" }, { "startMs": 1137190, "text": "we are starting a project? How do you" }, { "startMs": 1137200, "text": "we are starting a project? How do you plan these verification costs?" }, { "startMs": 1139510, "text": "plan these verification costs?" }, { "startMs": 1139520, "text": "plan these verification costs? Yeah. So the the way that we do this is" }, { "startMs": 1142070, "text": "Yeah. So the the way that we do this is" }, { "startMs": 1142080, "text": "Yeah. So the the way that we do this is like we give people the tools to measure" }, { "startMs": 1144390, "text": "like we give people the tools to measure" }, { "startMs": 1144400, "text": "like we give people the tools to measure like what's your cost per PR and then" }, { "startMs": 1147190, "text": "like what's your cost per PR and then" }, { "startMs": 1147200, "text": "like what's your cost per PR and then what percent of the cost is going" }, { "startMs": 1148630, "text": "what percent of the cost is going" }, { "startMs": 1148640, "text": "what percent of the cost is going towards like scoring or benchmarking or" }, { "startMs": 1150470, "text": "towards like scoring or benchmarking or" }, { "startMs": 1150480, "text": "towards like scoring or benchmarking or stuff like that. Uh and then you can" }, { "startMs": 1152630, "text": "stuff like that. Uh and then you can" }, { "startMs": 1152640, "text": "stuff like that. Uh and then you can kind of extrapolate it out like are you" }, { "startMs": 1154549, "text": "kind of extrapolate it out like are you" }, { "startMs": 1154559, "text": "kind of extrapolate it out like are you willing to spend x amount on a project" }, { "startMs": 1156950, "text": "willing to spend x amount on a project" }, { "startMs": 1156960, "text": "willing to spend x amount on a project or whatever. But the the key the key" }, { "startMs": 1159510, "text": "or whatever. But the the key the key" }, { "startMs": 1159520, "text": "or whatever. But the the key the key thing um that that I want people to" }, { "startMs": 1161669, "text": "thing um that that I want people to" }, { "startMs": 1161679, "text": "thing um that that I want people to leave if there's one idea is like really" }, { "startMs": 1164230, "text": "leave if there's one idea is like really" }, { "startMs": 1164240, "text": "leave if there's one idea is like really set up something that you can measure" }, { "startMs": 1165590, "text": "set up something that you can measure" }, { "startMs": 1165600, "text": "set up something that you can measure and improve. And once you can truly" }, { "startMs": 1167669, "text": "and improve. And once you can truly" }, { "startMs": 1167679, "text": "and improve. And once you can truly measure in a closed loop system, you can" }, { "startMs": 1170150, "text": "measure in a closed loop system, you can" }, { "startMs": 1170160, "text": "measure in a closed loop system, you can you can do all of these things in a way" }, { "startMs": 1171590, "text": "you can do all of these things in a way" }, { "startMs": 1171600, "text": "you can do all of these things in a way that's just never been possible before." }, { "startMs": 1176640, "text": "Any other questions?" }, { "startMs": 1178150, "text": "Any other questions?" }, { "startMs": 1178160, "text": "Any other questions? Question in the back." }, { "startMs": 1178870, "text": "Question in the back." }, { "startMs": 1178880, "text": "Question in the back. Ycess." }, { "startMs": 1188960, "text": "Yeah. So, so the question was around uh" }, { "startMs": 1191830, "text": "Yeah. So, so the question was around uh" }, { "startMs": 1191840, "text": "Yeah. So, so the question was around uh how do we do cost per PR? So this is" }, { "startMs": 1193830, "text": "how do we do cost per PR? So this is" }, { "startMs": 1193840, "text": "how do we do cost per PR? So this is data curation" }, { "startMs": 1194870, "text": "data curation" }, { "startMs": 1194880, "text": "data curation the the data curation" }, { "startMs": 1197669, "text": "the the data curation" }, { "startMs": 1197679, "text": "the the data curation um are you so I just want to make sure" }, { "startMs": 1199750, "text": "um are you so I just want to make sure" }, { "startMs": 1199760, "text": "um are you so I just want to make sure we're talking about the same thing. So" }, { "startMs": 1200950, "text": "we're talking about the same thing. So" }, { "startMs": 1200960, "text": "we're talking about the same thing. So you're talking about like when I" }, { "startMs": 1203590, "text": "you're talking about like when I" }, { "startMs": 1203600, "text": "you're talking about like when I for the the benchmarking or for like our" }, { "startMs": 1206390, "text": "for the the benchmarking or for like our" }, { "startMs": 1206400, "text": "for the the benchmarking or for like our general" }, { "startMs": 1210320, "text": "Okay. Yeah. So the way that we we do" }, { "startMs": 1212549, "text": "Okay. Yeah. So the way that we we do" }, { "startMs": 1212559, "text": "Okay. Yeah. So the way that we we do this is like you can either build up a" }, { "startMs": 1214390, "text": "this is like you can either build up a" }, { "startMs": 1214400, "text": "this is like you can either build up a benchmark set by hand which a lot of" }, { "startMs": 1216710, "text": "benchmark set by hand which a lot of" }, { "startMs": 1216720, "text": "benchmark set by hand which a lot of people will do. They'll you know and" }, { "startMs": 1218150, "text": "people will do. They'll you know and" }, { "startMs": 1218160, "text": "people will do. They'll you know and that's kind of what we recommend to" }, { "startMs": 1219430, "text": "that's kind of what we recommend to" }, { "startMs": 1219440, "text": "that's kind of what we recommend to start. uh we will pick like for ours we" }, { "startMs": 1222390, "text": "start. uh we will pick like for ours we" }, { "startMs": 1222400, "text": "start. uh we will pick like for ours we pick like a set of sort of small medium" }, { "startMs": 1224630, "text": "pick like a set of sort of small medium" }, { "startMs": 1224640, "text": "pick like a set of sort of small medium hard tasks whatever you want uh to be" }, { "startMs": 1227669, "text": "hard tasks whatever you want uh to be" }, { "startMs": 1227679, "text": "hard tasks whatever you want uh to be representative or you can actually ask" }, { "startMs": 1230950, "text": "representative or you can actually ask" }, { "startMs": 1230960, "text": "representative or you can actually ask the factory to do it because the factory" }, { "startMs": 1233029, "text": "the factory to do it because the factory" }, { "startMs": 1233039, "text": "the factory to do it because the factory is aware of all the past sessions and so" }, { "startMs": 1235350, "text": "is aware of all the past sessions and so" }, { "startMs": 1235360, "text": "is aware of all the past sessions and so and everything is stored in code so it's" }, { "startMs": 1237909, "text": "and everything is stored in code so it's" }, { "startMs": 1237919, "text": "and everything is stored in code so it's possible to have the factory itself look" }, { "startMs": 1240230, "text": "possible to have the factory itself look" }, { "startMs": 1240240, "text": "possible to have the factory itself look at prior sessions and identify" }, { "startMs": 1242710, "text": "at prior sessions and identify" }, { "startMs": 1242720, "text": "at prior sessions and identify interesting ones for a benchmark set" }, { "startMs": 1244710, "text": "interesting ones for a benchmark set" }, { "startMs": 1244720, "text": "interesting ones for a benchmark set which is pretty cool" }, { "startMs": 1247110, "text": "which is pretty cool" }, { "startMs": 1247120, "text": "which is pretty cool y" }, { "startMs": 1247590, "text": "y" }, { "startMs": 1247600, "text": "y how is correctness evaluated on chart." }, { "startMs": 1250630, "text": "how is correctness evaluated on chart." }, { "startMs": 1250640, "text": "how is correctness evaluated on chart. Yeah. So everything in this chart is" }, { "startMs": 1252630, "text": "Yeah. So everything in this chart is" }, { "startMs": 1252640, "text": "Yeah. So everything in this chart is driven by LLM as a judge which we we may" }, { "startMs": 1256549, "text": "driven by LLM as a judge which we we may" }, { "startMs": 1256559, "text": "driven by LLM as a judge which we we may I think another way of doing it is human" }, { "startMs": 1258149, "text": "I think another way of doing it is human" }, { "startMs": 1258159, "text": "I think another way of doing it is human grading which makes total sense. We just" }, { "startMs": 1259909, "text": "grading which makes total sense. We just" }, { "startMs": 1259919, "text": "grading which makes total sense. We just haven't added that yet. So I would" }, { "startMs": 1261190, "text": "haven't added that yet. So I would" }, { "startMs": 1261200, "text": "haven't added that yet. So I would suggest that as well. But basically" }, { "startMs": 1263830, "text": "suggest that as well. But basically" }, { "startMs": 1263840, "text": "suggest that as well. But basically anything that is defined uh in the" }, { "startMs": 1266789, "text": "anything that is defined uh in the" }, { "startMs": 1266799, "text": "anything that is defined uh in the system as a scorer. So you define these" }, { "startMs": 1269750, "text": "system as a scorer. So you define these" }, { "startMs": 1269760, "text": "system as a scorer. So you define these scores which basically grade how well" }, { "startMs": 1271750, "text": "scores which basically grade how well" }, { "startMs": 1271760, "text": "scores which basically grade how well something is doing can be used as a" }, { "startMs": 1273270, "text": "something is doing can be used as a" }, { "startMs": 1273280, "text": "something is doing can be used as a benchmark evaluation criteria." }, { "startMs": 1277029, "text": "benchmark evaluation criteria." }, { "startMs": 1277039, "text": "benchmark evaluation criteria. Yeah." }, { "startMs": 1278710, "text": "Yeah." }, { "startMs": 1278720, "text": "Yeah. this work platform is this open source" }, { "startMs": 1281510, "text": "this work platform is this open source" }, { "startMs": 1281520, "text": "this work platform is this open source or is this only for enterprise?" }, { "startMs": 1283750, "text": "or is this only for enterprise?" }, { "startMs": 1283760, "text": "or is this only for enterprise? So it is is not open source. It's it's" }, { "startMs": 1287750, "text": "So it is is not open source. It's it's" }, { "startMs": 1287760, "text": "So it is is not open source. It's it's like very flexible and like like the" }, { "startMs": 1289990, "text": "like very flexible and like like the" }, { "startMs": 1290000, "text": "like very flexible and like like the idea is to make it open so you can bring" }, { "startMs": 1291909, "text": "idea is to make it open so you can bring" }, { "startMs": 1291919, "text": "idea is to make it open so you can bring any existing tools into it. It's not" }, { "startMs": 1294630, "text": "any existing tools into it. It's not" }, { "startMs": 1294640, "text": "any existing tools into it. It's not only for enterprise like this is for" }, { "startMs": 1296789, "text": "only for enterprise like this is for" }, { "startMs": 1296799, "text": "only for enterprise like this is for small teams and you can try it out on" }, { "startMs": 1298470, "text": "small teams and you can try it out on" }, { "startMs": 1298480, "text": "small teams and you can try it out on your personal project. It is best for" }, { "startMs": 1301029, "text": "your personal project. It is best for" }, { "startMs": 1301039, "text": "your personal project. It is best for teams though I will say because like if" }, { "startMs": 1303590, "text": "teams though I will say because like if" }, { "startMs": 1303600, "text": "teams though I will say because like if you're um if you're an it's just" }, { "startMs": 1305510, "text": "you're um if you're an it's just" }, { "startMs": 1305520, "text": "you're um if you're an it's just overkill for an individual like you're" }, { "startMs": 1307029, "text": "overkill for an individual like you're" }, { "startMs": 1307039, "text": "overkill for an individual like you're probably gonna want to use like your" }, { "startMs": 1307990, "text": "probably gonna want to use like your" }, { "startMs": 1308000, "text": "probably gonna want to use like your cloud code sub or something but once you" }, { "startMs": 1310230, "text": "cloud code sub or something but once you" }, { "startMs": 1310240, "text": "cloud code sub or something but once you have a team of like maybe 10 or more" }, { "startMs": 1311750, "text": "have a team of like maybe 10 or more" }, { "startMs": 1311760, "text": "have a team of like maybe 10 or more engineers having a system like this is" }, { "startMs": 1313430, "text": "engineers having a system like this is" }, { "startMs": 1313440, "text": "engineers having a system like this is very valuable." } ]